From 201edaae83959bb9b11ee670061d533c76a90ae7 Mon Sep 17 00:00:00 2001
From: bryce13950 <bryce13950@gmail.com>
Date: Tue, 31 Dec 2024 02:22:15 +0000
Subject: [PATCH] =?UTF-8?q?Deploying=20to=20gh-pages=20from=20@=20Transfor?=
 =?UTF-8?q?merLensOrg/TransformerLens@f103debd1084cd79969164ac98ed9059a863?=
 =?UTF-8?q?54bc=20=F0=9F=9A=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../generated/model_properties_table.md.txt   |    9 +-
 ...7285d613390727b_can_be_used_as_mlp_py.html |    4 +-
 .../d_37285d613390727b_gated_mlp_4bit_py.html |    4 +-
 .../d_37285d613390727b_gated_mlp_py.html      |    4 +-
 .../coverage/d_37285d613390727b_mlp_py.html   |    4 +-
 .../coverage/d_37285d613390727b_moe_py.html   |    4 +-
 ...bfb219_activation_function_factory_py.html |    4 +-
 .../d_65d4430f90bfb219_mlp_factory_py.html    |    4 +-
 .../d_712808f24eb400fe___init___py.html       |    4 +-
 ...d_af97b5493da09a14_ActivationCache_py.html |    4 +-
 .../d_af97b5493da09a14_FactoredMatrix_py.html |    4 +-
 ...7b5493da09a14_HookedEncoderDecoder_py.html |    4 +-
 .../d_af97b5493da09a14_HookedEncoder_py.html  |    4 +-
 ...493da09a14_HookedTransformerConfig_py.html |   12 +-
 ...af97b5493da09a14_HookedTransformer_py.html | 1105 +++---
 .../d_af97b5493da09a14_SVDInterpreter_py.html |    4 +-
 .../d_af97b5493da09a14___init___py.html       |    4 +-
 .../coverage/d_af97b5493da09a14_evals_py.html |    4 +-
 .../d_af97b5493da09a14_head_detector_py.html  |    4 +-
 .../d_af97b5493da09a14_hook_points_py.html    |    4 +-
 ...493da09a14_loading_from_pretrained_py.html | 3503 +++++++++--------
 ...5493da09a14_past_key_value_caching_py.html |    4 +-
 .../d_af97b5493da09a14_patching_py.html       |    4 +-
 .../coverage/d_af97b5493da09a14_train_py.html |    4 +-
 .../coverage/d_af97b5493da09a14_utils_py.html |    4 +-
 .../d_b2114f845e0399b7___init___py.html       |    4 +-
 ...14f845e0399b7_activation_functions_py.html |    4 +-
 .../coverage/d_b2114f845e0399b7_addmm_py.html |    4 +-
 .../d_b2114f845e0399b7_attention_py.html      |    4 +-
 .../d_b2114f845e0399b7_devices_py.html        |    4 +-
 .../d_c1ea89878f9b2ac7___init___py.html       |    4 +-
 .../coverage/d_c1ea89878f9b2ac7_bert_py.html  |    4 +-
 .../coverage/d_c1ea89878f9b2ac7_bloom_py.html |    4 +-
 .../coverage/d_c1ea89878f9b2ac7_coder_py.html |    4 +-
 .../coverage/d_c1ea89878f9b2ac7_gemma_py.html |    4 +-
 .../coverage/d_c1ea89878f9b2ac7_gpt2_py.html  |    4 +-
 .../coverage/d_c1ea89878f9b2ac7_gptj_py.html  |    4 +-
 .../coverage/d_c1ea89878f9b2ac7_llama_py.html |    4 +-
 .../d_c1ea89878f9b2ac7_mingpt_py.html         |    4 +-
 .../d_c1ea89878f9b2ac7_mistral_py.html        |    4 +-
 .../d_c1ea89878f9b2ac7_mixtral_py.html        |    4 +-
 .../d_c1ea89878f9b2ac7_nanogpt_py.html        |    4 +-
 .../d_c1ea89878f9b2ac7_neel_solu_old_py.html  |    4 +-
 .../coverage/d_c1ea89878f9b2ac7_neo_py.html   |    4 +-
 .../coverage/d_c1ea89878f9b2ac7_neox_py.html  |    4 +-
 .../coverage/d_c1ea89878f9b2ac7_opt_py.html   |    4 +-
 .../coverage/d_c1ea89878f9b2ac7_phi3_py.html  |    4 +-
 .../coverage/d_c1ea89878f9b2ac7_phi_py.html   |    4 +-
 .../coverage/d_c1ea89878f9b2ac7_qwen2_py.html |   84 +-
 .../coverage/d_c1ea89878f9b2ac7_qwen_py.html  |    4 +-
 .../coverage/d_c1ea89878f9b2ac7_t5_py.html    |    4 +-
 .../d_db46118ef83ad831___init___py.html       |    4 +-
 ...b46118ef83ad831_abstract_attention_py.html |    4 +-
 .../d_db46118ef83ad831_attention_py.html      |    4 +-
 .../d_db46118ef83ad831_bert_block_py.html     |    4 +-
 .../d_db46118ef83ad831_bert_embed_py.html     |    4 +-
 .../d_db46118ef83ad831_bert_mlm_head_py.html  |    4 +-
 .../coverage/d_db46118ef83ad831_embed_py.html |    4 +-
 ...8ef83ad831_grouped_query_attention_py.html |    4 +-
 .../d_db46118ef83ad831_layer_norm_pre_py.html |    4 +-
 .../d_db46118ef83ad831_layer_norm_py.html     |    4 +-
 .../d_db46118ef83ad831_pos_embed_py.html      |    4 +-
 .../d_db46118ef83ad831_rms_norm_pre_py.html   |    4 +-
 .../d_db46118ef83ad831_rms_norm_py.html       |    4 +-
 .../d_db46118ef83ad831_t5_attention_py.html   |    4 +-
 .../d_db46118ef83ad831_t5_block_py.html       |    4 +-
 ...db46118ef83ad831_token_typed_embed_py.html |    4 +-
 ...db46118ef83ad831_transformer_block_py.html |   18 +-
 .../d_db46118ef83ad831_unembed_py.html        |    4 +-
 _static/coverage/index.html                   |   48 +-
 _static/coverage/status.json                  |    2 +-
 ...nsformer_lens.loading_from_pretrained.html |    2 +-
 .../demos/Exploratory_Analysis_Demo.html      |  102 +-
 .../demos/Exploratory_Analysis_Demo.ipynb     | 1814 ++++-----
 generated/demos/Main_Demo.html                |  107 +-
 generated/demos/Main_Demo.ipynb               | 2447 ++++++------
 generated/model_properties_table.html         |  264 +-
 searchindex.js                                |    2 +-
 78 files changed, 4893 insertions(+), 4878 deletions(-)

diff --git a/_sources/generated/model_properties_table.md.txt b/_sources/generated/model_properties_table.md.txt
index b653ccf5a..66e6ebe90 100644
--- a/_sources/generated/model_properties_table.md.txt
+++ b/_sources/generated/model_properties_table.md.txt
@@ -109,14 +109,15 @@
 | meta-llama/Meta-Llama-3-8B-Instruct  | 7.8B       |         32 |      4096 |        32 | silu              |    8192 |    128256 |      128 |   14336 | 8                   |
 | meta-llama/Meta-Llama-3-70B          | 78B        |         80 |      8192 |        64 | silu              |    8192 |    128256 |      128 |   28672 | 8                   |
 | meta-llama/Meta-Llama-3-70B-Instruct | 78B        |         80 |      8192 |        64 | silu              |    8192 |    128256 |      128 |   28672 | 8                   |
-| meta-llama/Llama-3.2-1B              | 1.1B       |         16 |      2048 |        32 | silu              |    2048 |    128256 |       64 |    8192 | 8                   |
-| meta-llama/Llama-3.2-3B              | 3.2B       |         28 |      3072 |        24 | silu              |    2048 |    128256 |      128 |    8192 | 8                   |
-| meta-llama/Llama-3.2-1B-Instruct     | 1.1B       |         16 |      2048 |        32 | silu              |    2048 |    128256 |       64 |    8192 | 8                   |
-| meta-llama/Llama-3.2-3B-Instruct     | 3.2B       |         28 |      3072 |        24 | silu              |    2048 |    128256 |      128 |    8192 | 8                   |
 | meta-llama/Llama-3.1-70B             | 78B        |         80 |      8192 |        64 | silu              |    2048 |    128256 |      128 |   28672 | 8                   |
 | meta-llama/Llama-3.1-8B              | 7.8B       |         32 |      4096 |        32 | silu              |    2048 |    128256 |      128 |   14336 | 8                   |
 | meta-llama/Llama-3.1-8B-Instruct     | 7.8B       |         32 |      4096 |        32 | silu              |    2048 |    128256 |      128 |   14336 | 8                   |
 | meta-llama/Llama-3.1-70B-Instruct    | 78B        |         80 |      8192 |        64 | silu              |    2048 |    128256 |      128 |   28672 | 8                   |
+| meta-llama/Llama-3.2-1B              | 1.1B       |         16 |      2048 |        32 | silu              |    2048 |    128256 |       64 |    8192 | 8                   |
+| meta-llama/Llama-3.2-3B              | 3.2B       |         28 |      3072 |        24 | silu              |    2048 |    128256 |      128 |    8192 | 8                   |
+| meta-llama/Llama-3.2-1B-Instruct     | 1.1B       |         16 |      2048 |        32 | silu              |    2048 |    128256 |       64 |    8192 | 8                   |
+| meta-llama/Llama-3.2-3B-Instruct     | 3.2B       |         28 |      3072 |        24 | silu              |    2048 |    128256 |      128 |    8192 | 8                   |
+| meta-llama/Llama-3.3-70B-Instruct    | 78B        |         80 |      8192 |        64 | silu              |    2048 |    128256 |      128 |   28672 | 8                   |
 | othello-gpt                          | 25M        |          8 |       512 |         8 | gelu              |      59 |        61 |       64 |    2048 |                     |
 | bert-base-cased                      | 85M        |         12 |       768 |        12 | gelu              |     512 |     28996 |       64 |    3072 |                     |
 | tiny-stories-1M                      | 393K       |          8 |        64 |        16 | gelu              |    2048 |     50257 |        4 |     256 |                     |
diff --git a/_static/coverage/d_37285d613390727b_can_be_used_as_mlp_py.html b/_static/coverage/d_37285d613390727b_can_be_used_as_mlp_py.html
index bfe5dd961..fa8b40624 100644
--- a/_static/coverage/d_37285d613390727b_can_be_used_as_mlp_py.html
+++ b/_static/coverage/d_37285d613390727b_can_be_used_as_mlp_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_gated_mlp_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -166,7 +166,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_gated_mlp_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_37285d613390727b_gated_mlp_4bit_py.html b/_static/coverage/d_37285d613390727b_gated_mlp_4bit_py.html
index c1488aa23..99742372b 100644
--- a/_static/coverage/d_37285d613390727b_gated_mlp_4bit_py.html
+++ b/_static/coverage/d_37285d613390727b_gated_mlp_4bit_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_mlp_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -168,7 +168,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_mlp_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_37285d613390727b_gated_mlp_py.html b/_static/coverage/d_37285d613390727b_gated_mlp_py.html
index 9638d4f3e..bf9b5ace6 100644
--- a/_static/coverage/d_37285d613390727b_gated_mlp_py.html
+++ b/_static/coverage/d_37285d613390727b_gated_mlp_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_gated_mlp_4bit_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -164,7 +164,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_gated_mlp_4bit_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_37285d613390727b_mlp_py.html b/_static/coverage/d_37285d613390727b_mlp_py.html
index cb99580ff..46d90bd6f 100644
--- a/_static/coverage/d_37285d613390727b_mlp_py.html
+++ b/_static/coverage/d_37285d613390727b_mlp_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_moe_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -140,7 +140,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_moe_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_37285d613390727b_moe_py.html b/_static/coverage/d_37285d613390727b_moe_py.html
index 2f9b7588f..cd0fbd0f7 100644
--- a/_static/coverage/d_37285d613390727b_moe_py.html
+++ b/_static/coverage/d_37285d613390727b_moe_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_pos_embed_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -204,7 +204,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_pos_embed_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_65d4430f90bfb219_activation_function_factory_py.html b/_static/coverage/d_65d4430f90bfb219_activation_function_factory_py.html
index f337a8c69..7b3cd9df6 100644
--- a/_static/coverage/d_65d4430f90bfb219_activation_function_factory_py.html
+++ b/_static/coverage/d_65d4430f90bfb219_activation_function_factory_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_65d4430f90bfb219_mlp_factory_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -128,7 +128,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_65d4430f90bfb219_mlp_factory_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_65d4430f90bfb219_mlp_factory_py.html b/_static/coverage/d_65d4430f90bfb219_mlp_factory_py.html
index 39e888b6e..1c785a1a8 100644
--- a/_static/coverage/d_65d4430f90bfb219_mlp_factory_py.html
+++ b/_static/coverage/d_65d4430f90bfb219_mlp_factory_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_head_detector_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -112,7 +112,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_head_detector_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_712808f24eb400fe___init___py.html b/_static/coverage/d_712808f24eb400fe___init___py.html
index b3b5558a1..e134b7776 100644
--- a/_static/coverage/d_712808f24eb400fe___init___py.html
+++ b/_static/coverage/d_712808f24eb400fe___init___py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7___init___py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -91,7 +91,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7___init___py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_ActivationCache_py.html b/_static/coverage/d_af97b5493da09a14_ActivationCache_py.html
index 444524681..67001a653 100644
--- a/_static/coverage/d_af97b5493da09a14_ActivationCache_py.html
+++ b/_static/coverage/d_af97b5493da09a14_ActivationCache_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_FactoredMatrix_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -1196,7 +1196,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_FactoredMatrix_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_FactoredMatrix_py.html b/_static/coverage/d_af97b5493da09a14_FactoredMatrix_py.html
index 14d2e7f07..2dcf3971d 100644
--- a/_static/coverage/d_af97b5493da09a14_FactoredMatrix_py.html
+++ b/_static/coverage/d_af97b5493da09a14_FactoredMatrix_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_HookedEncoder_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -365,7 +365,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_HookedEncoder_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_HookedEncoderDecoder_py.html b/_static/coverage/d_af97b5493da09a14_HookedEncoderDecoder_py.html
index 90f3ee72e..6c8395dfa 100644
--- a/_static/coverage/d_af97b5493da09a14_HookedEncoderDecoder_py.html
+++ b/_static/coverage/d_af97b5493da09a14_HookedEncoderDecoder_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_HookedTransformer_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -507,7 +507,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_HookedTransformer_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_HookedEncoder_py.html b/_static/coverage/d_af97b5493da09a14_HookedEncoder_py.html
index f7f95782c..f3166c01d 100644
--- a/_static/coverage/d_af97b5493da09a14_HookedEncoder_py.html
+++ b/_static/coverage/d_af97b5493da09a14_HookedEncoder_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_HookedEncoderDecoder_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -461,7 +461,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_HookedEncoderDecoder_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_HookedTransformerConfig_py.html b/_static/coverage/d_af97b5493da09a14_HookedTransformerConfig_py.html
index 112fab43f..250b37bfb 100644
--- a/_static/coverage/d_af97b5493da09a14_HookedTransformerConfig_py.html
+++ b/_static/coverage/d_af97b5493da09a14_HookedTransformerConfig_py.html
@@ -2,7 +2,7 @@
 <html>
 <head>
     <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
-    <title>Coverage for transformer_lens/HookedTransformerConfig.py: 91%</title>
+    <title>Coverage for transformer_lens/HookedTransformerConfig.py: 92%</title>
     <link rel="icon" sizes="32x32" href="favicon_32.png">
     <link rel="stylesheet" href="style.css" type="text/css">
     <script type="text/javascript" src="coverage_html.js" defer></script>
@@ -12,7 +12,7 @@
     <div class="content">
         <h1>
             <span class="text">Coverage for </span><b>transformer_lens/HookedTransformerConfig.py</b>:
-            <span class="pc_cov">91%</span>
+            <span class="pc_cov">92%</span>
         </h1>
         <aside id="help_panel_wrapper">
             <input id="help_panel_state" type="checkbox">
@@ -59,7 +59,7 @@ <h2>
             <button type="button" class="run button_toggle_run" value="run" data-shortcut="r" title="Toggle lines run">127<span class="text"> run</span></button>
             <button type="button" class="mis show_mis button_toggle_mis" value="mis" data-shortcut="m" title="Toggle lines missing">8<span class="text"> missing</span></button>
             <button type="button" class="exc show_exc button_toggle_exc" value="exc" data-shortcut="x" title="Toggle lines excluded">0<span class="text"> excluded</span></button>
-            <button type="button" class="par run show_par button_toggle_par" value="par" data-shortcut="p" title="Toggle lines partially run">7<span class="text"> partial</span></button>
+            <button type="button" class="par run show_par button_toggle_par" value="par" data-shortcut="p" title="Toggle lines partially run">6<span class="text"> partial</span></button>
         </h2>
         <p class="text">
             <a id="prevFileLink" class="nav" href="d_af97b5493da09a14_HookedTransformer_py.html">&#xab; prev</a> &nbsp; &nbsp;
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_SVDInterpreter_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -373,7 +373,7 @@ <h2>
     <p class="run"><span class="n"><a id="t289" href="#t289">289</a></span><span class="t">            <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t290" href="#t290">290</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">act_fn</span> <span class="key">in</span> <span class="nam">SUPPORTED_ACTIVATIONS</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t291" href="#t291">291</a></span><span class="t">            <span class="op">)</span><span class="op">,</span> <span class="str">f"act_fn={self.act_fn} must be one of {SUPPORTED_ACTIVATIONS}"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t292" href="#t292">292</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">initializer_range</span> <span class="op">&lt;</span> <span class="num">0</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">init_mode</span> <span class="op">==</span> <span class="str">"gpt2"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">292&#x202F;&#x219B;&#x202F;295</span><span class="annotate long">line 292 didn't jump to line 295, because the condition on line 292 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t292" href="#t292">292</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">initializer_range</span> <span class="op">&lt;</span> <span class="num">0</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">init_mode</span> <span class="op">==</span> <span class="str">"gpt2"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t293" href="#t293">293</a></span><span class="t">            <span class="com"># Roughly copy the GPT-2 value, but proportional to sqrt(1/d_model)</span>&nbsp;</span><span class="r"></span></p>
     <p class="run"><span class="n"><a id="t294" href="#t294">294</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">initializer_range</span> <span class="op">=</span> <span class="num">0.8</span> <span class="op">/</span> <span class="nam">np</span><span class="op">.</span><span class="nam">sqrt</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">d_model</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="par run show_par"><span class="n"><a id="t295" href="#t295">295</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">initializer_range</span> <span class="op">&lt;</span> <span class="num">0</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">init_mode</span> <span class="op">!=</span> <span class="str">"gpt2"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">295&#x202F;&#x219B;&#x202F;297</span><span class="annotate long">line 295 didn't jump to line 297, because the condition on line 295 was never true</span></span></p>
@@ -463,7 +463,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_SVDInterpreter_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_HookedTransformer_py.html b/_static/coverage/d_af97b5493da09a14_HookedTransformer_py.html
index 6dd470c00..a78676d91 100644
--- a/_static/coverage/d_af97b5493da09a14_HookedTransformer_py.html
+++ b/_static/coverage/d_af97b5493da09a14_HookedTransformer_py.html
@@ -2,7 +2,7 @@
 <html>
 <head>
     <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
-    <title>Coverage for transformer_lens/HookedTransformer.py: 75%</title>
+    <title>Coverage for transformer_lens/HookedTransformer.py: 77%</title>
     <link rel="icon" sizes="32x32" href="favicon_32.png">
     <link rel="stylesheet" href="style.css" type="text/css">
     <script type="text/javascript" src="coverage_html.js" defer></script>
@@ -12,7 +12,7 @@
     <div class="content">
         <h1>
             <span class="text">Coverage for </span><b>transformer_lens/HookedTransformer.py</b>:
-            <span class="pc_cov">75%</span>
+            <span class="pc_cov">77%</span>
         </h1>
         <aside id="help_panel_wrapper">
             <input id="help_panel_state" type="checkbox">
@@ -55,11 +55,11 @@ <h1>
             </div>
         </aside>
         <h2>
-            <span class="text">740 statements &nbsp;</span>
-            <button type="button" class="run button_toggle_run" value="run" data-shortcut="r" title="Toggle lines run">599<span class="text"> run</span></button>
-            <button type="button" class="mis show_mis button_toggle_mis" value="mis" data-shortcut="m" title="Toggle lines missing">141<span class="text"> missing</span></button>
+            <span class="text">741 statements &nbsp;</span>
+            <button type="button" class="run button_toggle_run" value="run" data-shortcut="r" title="Toggle lines run">609<span class="text"> run</span></button>
+            <button type="button" class="mis show_mis button_toggle_mis" value="mis" data-shortcut="m" title="Toggle lines missing">132<span class="text"> missing</span></button>
             <button type="button" class="exc show_exc button_toggle_exc" value="exc" data-shortcut="x" title="Toggle lines excluded">0<span class="text"> excluded</span></button>
-            <button type="button" class="par run show_par button_toggle_par" value="par" data-shortcut="p" title="Toggle lines partially run">90<span class="text"> partial</span></button>
+            <button type="button" class="par run show_par button_toggle_par" value="par" data-shortcut="p" title="Toggle lines partially run">79<span class="text"> partial</span></button>
         </h2>
         <p class="text">
             <a id="prevFileLink" class="nav" href="d_af97b5493da09a14_HookedEncoderDecoder_py.html">&#xab; prev</a> &nbsp; &nbsp;
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_HookedTransformerConfig_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -275,8 +275,8 @@ <h2>
     <p class="pln"><span class="n"><a id="t191" href="#t191">191</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
     <p class="par run show_par"><span class="n"><a id="t192" href="#t192">192</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">==</span> <span class="str">"RMS"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">192&#x202F;&#x219B;&#x202F;193</span><span class="annotate long">line 192 didn't jump to line 193, because the condition on line 192 was never true</span></span></p>
     <p class="mis show_mis"><span class="n"><a id="t193" href="#t193">193</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">ln_final</span> <span class="op">=</span> <span class="nam">RMSNorm</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t194" href="#t194">194</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">==</span> <span class="str">"RMSPre"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">194&#x202F;&#x219B;&#x202F;195</span><span class="annotate long">line 194 didn't jump to line 195, because the condition on line 194 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t195" href="#t195">195</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">ln_final</span> <span class="op">=</span> <span class="nam">RMSNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t194" href="#t194">194</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">==</span> <span class="str">"RMSPre"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t195" href="#t195">195</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">ln_final</span> <span class="op">=</span> <span class="nam">RMSNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="run"><span class="n"><a id="t196" href="#t196">196</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">==</span> <span class="str">"LN"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
     <p class="par run show_par"><span class="n"><a id="t197" href="#t197">197</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">final_rms</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">197&#x202F;&#x219B;&#x202F;198</span><span class="annotate long">line 197 didn't jump to line 198, because the condition on line 197 was never true</span></span></p>
     <p class="mis show_mis"><span class="n"><a id="t198" href="#t198">198</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">ln_final</span> <span class="op">=</span> <span class="nam">RMSNorm</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
@@ -786,8 +786,8 @@ <h2>
     <p class="mis show_mis"><span class="n"><a id="t702" href="#t702">702</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">eos_token</span> <span class="op">=</span> <span class="str">"&lt;|endoftext|>"</span>&nbsp;</span><span class="r"></span></p>
     <p class="run"><span class="n"><a id="t703" href="#t703">703</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">pad_token</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
     <p class="run"><span class="n"><a id="t704" href="#t704">704</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">pad_token</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">eos_token</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t705" href="#t705">705</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">bos_token</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">705&#x202F;&#x219B;&#x202F;706</span><span class="annotate long">line 705 didn't jump to line 706, because the condition on line 705 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t706" href="#t706">706</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">bos_token</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">eos_token</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t705" href="#t705">705</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">bos_token</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t706" href="#t706">706</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">bos_token</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">eos_token</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t707" href="#t707">707</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t708" href="#t708">708</a></span><span class="t">        <span class="com"># Infer vocab size from tokenizer</span>&nbsp;</span><span class="r"></span></p>
     <p class="run"><span class="n"><a id="t709" href="#t709">709</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_vocab</span> <span class="op">==</span> <span class="op">-</span><span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
@@ -1395,7 +1395,7 @@ <h2>
     <p class="run"><span class="n"><a id="t1311" href="#t1311">1311</a></span><span class="t">                <span class="nam">center_writing_weights</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
     <p class="par run show_par"><span class="n"><a id="t1312" href="#t1312">1312</a></span><span class="t">        <span class="key">if</span> <span class="nam">center_unembed</span> <span class="key">and</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">output_logits_soft_cap</span> <span class="op">></span> <span class="num">0.0</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1312&#x202F;&#x219B;&#x202F;1313</span><span class="annotate long">line 1312 didn't jump to line 1313, because the condition on line 1312 was never true</span></span></p>
     <p class="mis show_mis"><span class="n"><a id="t1313" href="#t1313">1313</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1314" href="#t1314">1314</a></span><span class="t">                <span class="str">"You tried to specify center_unembed=True for a model using logit softcap, but this can't be done! Softcapping is not invariant upon adding a constant"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1314" href="#t1314">1314</a></span><span class="t">                <span class="str">"You tried to specify center_unembed=True for a model using logit softcap, but this can't be done! Softcapping is not invariant upon adding a constant "</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1315" href="#t1315">1315</a></span><span class="t">                <span class="str">"Setting center_unembed=False instead."</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1316" href="#t1316">1316</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="mis show_mis"><span class="n"><a id="t1317" href="#t1317">1317</a></span><span class="t">            <span class="nam">center_unembed</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
@@ -1644,10 +1644,10 @@ <h2>
     <p class="mis show_mis"><span class="n"><a id="t1560" href="#t1560">1560</a></span><span class="t">                <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1561" href="#t1561">1561</a></span><span class="t">                    <span class="str">"You are using MoE, so the layer norm weights can't be folded! Skipping"</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1562" href="#t1562">1562</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1563" href="#t1563">1563</a></span><span class="t">            <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="key">in</span> <span class="op">[</span><span class="str">"LN"</span><span class="op">,</span> <span class="str">"LNPre"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1563&#x202F;&#x219B;&#x202F;1565</span><span class="annotate long">line 1563 didn't jump to line 1565, because the condition on line 1563 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t1563" href="#t1563">1563</a></span><span class="t">            <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="key">in</span> <span class="op">[</span><span class="str">"LN"</span><span class="op">,</span> <span class="str">"LNPre"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
     <p class="run"><span class="n"><a id="t1564" href="#t1564">1564</a></span><span class="t">                <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">fold_layer_norm</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1565" href="#t1565">1565</a></span><span class="t">            <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="key">in</span> <span class="op">[</span><span class="str">"RMS"</span><span class="op">,</span> <span class="str">"RMSPre"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1566" href="#t1566">1566</a></span><span class="t">                <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">fold_layer_norm</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1565" href="#t1565">1565</a></span><span class="t">            <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="key">in</span> <span class="op">[</span><span class="str">"RMS"</span><span class="op">,</span> <span class="str">"RMSPre"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1565&#x202F;&#x219B;&#x202F;1570</span><span class="annotate long">line 1565 didn't jump to line 1570, because the condition on line 1565 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t1566" href="#t1566">1566</a></span><span class="t">                <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">fold_layer_norm</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1567" href="#t1567">1567</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">,</span> <span class="nam">fold_biases</span><span class="op">=</span><span class="key">False</span><span class="op">,</span> <span class="nam">center_weights</span><span class="op">=</span><span class="key">False</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1568" href="#t1568">1568</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1569" href="#t1569">1569</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
@@ -1656,8 +1656,8 @@ <h2>
     <p class="pln"><span class="n"><a id="t1572" href="#t1572">1572</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1573" href="#t1573">1573</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
     <p class="run"><span class="n"><a id="t1574" href="#t1574">1574</a></span><span class="t">        <span class="key">if</span> <span class="nam">center_writing_weights</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1575" href="#t1575">1575</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="key">not</span> <span class="key">in</span> <span class="op">[</span><span class="str">"LN"</span><span class="op">,</span> <span class="str">"LNPre"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1575&#x202F;&#x219B;&#x202F;1576</span><span class="annotate long">line 1575 didn't jump to line 1576, because the condition on line 1575 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1576" href="#t1576">1576</a></span><span class="t">                <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1575" href="#t1575">1575</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="key">not</span> <span class="key">in</span> <span class="op">[</span><span class="str">"LN"</span><span class="op">,</span> <span class="str">"LNPre"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1576" href="#t1576">1576</a></span><span class="t">                <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1577" href="#t1577">1577</a></span><span class="t">                    <span class="str">"You are not using LayerNorm, so the writing weights can't be centered! Skipping"</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1578" href="#t1578">1578</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="run"><span class="n"><a id="t1579" href="#t1579">1579</a></span><span class="t">            <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">final_rms</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
@@ -1713,7 +1713,7 @@ <h2>
     <p class="pln"><span class="n"><a id="t1629" href="#t1629">1629</a></span><span class="t">            <span class="com"># along every axis other than d_model. Each weight matrix right multiplies. To fold in</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1630" href="#t1630">1630</a></span><span class="t">            <span class="com"># the bias, we use the W_ matrix to map it to the hidden space of the layer, so we need</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1631" href="#t1631">1631</a></span><span class="t">            <span class="com"># to sum along axis -2, which is the residual stream space axis.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1632" href="#t1632">1632</a></span><span class="t">            <span class="key">if</span> <span class="nam">fold_biases</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1632&#x202F;&#x219B;&#x202F;1655</span><span class="annotate long">line 1632 didn't jump to line 1655</span></span></p>
+    <p class="run"><span class="n"><a id="t1632" href="#t1632">1632</a></span><span class="t">            <span class="key">if</span> <span class="nam">fold_biases</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
     <p class="run"><span class="n"><a id="t1633" href="#t1633">1633</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_Q"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_Q"</span><span class="op">]</span> <span class="op">+</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1634" href="#t1634">1634</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_Q"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1635" href="#t1635">1635</a></span><span class="t">                    <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln1.b"</span><span class="op">]</span><span class="op">[</span><span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
@@ -1754,7 +1754,7 @@ <h2>
     <p class="pln"><span class="n"><a id="t1670" href="#t1670">1670</a></span><span class="t">            <span class="com"># input vector of the matrix doesn't matter and can be set to zero. Equivalently, the</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1671" href="#t1671">1671</a></span><span class="t">            <span class="com"># output of LayerNormPre is orthogonal to the vector of all 1s (because dotting with</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1672" href="#t1672">1672</a></span><span class="t">            <span class="com"># that gets the sum), so we can remove the component of the matrix parallel to this.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1673" href="#t1673">1673</a></span><span class="t">            <span class="key">if</span> <span class="nam">center_weights</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1673&#x202F;&#x219B;&#x202F;1691</span><span class="annotate long">line 1673 didn't jump to line 1691, because the condition on line 1673 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t1673" href="#t1673">1673</a></span><span class="t">            <span class="key">if</span> <span class="nam">center_weights</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
     <p class="run"><span class="n"><a id="t1674" href="#t1674">1674</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_Q"</span><span class="op">]</span> <span class="op">-=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">reduce</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1675" href="#t1675">1675</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_Q"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1676" href="#t1676">1676</a></span><span class="t">                    <span class="str">"head_index d_model d_head -> head_index 1 d_head"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
@@ -1773,7 +1773,7 @@ <h2>
     <p class="pln"><span class="n"><a id="t1689" href="#t1689">1689</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1690" href="#t1690">1690</a></span><span class="t">            <span class="com"># Fold ln2 into MLP</span>&nbsp;</span><span class="r"></span></p>
     <p class="run"><span class="n"><a id="t1691" href="#t1691">1691</a></span><span class="t">            <span class="key">if</span> <span class="key">not</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">attn_only</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1692" href="#t1692">1692</a></span><span class="t">                <span class="key">if</span> <span class="nam">fold_biases</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1692&#x202F;&#x219B;&#x202F;1699</span><span class="annotate long">line 1692 didn't jump to line 1699</span></span></p>
+    <p class="run"><span class="n"><a id="t1692" href="#t1692">1692</a></span><span class="t">                <span class="key">if</span> <span class="nam">fold_biases</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
     <p class="run"><span class="n"><a id="t1693" href="#t1693">1693</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.b_in"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.b_in"</span><span class="op">]</span> <span class="op">+</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1694" href="#t1694">1694</a></span><span class="t">                        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_in"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1695" href="#t1695">1695</a></span><span class="t">                        <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln2.b"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
@@ -1784,15 +1784,15 @@ <h2>
     <p class="pln"><span class="n"><a id="t1700" href="#t1700">1700</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_in"</span><span class="op">]</span> <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln2.w"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1701" href="#t1701">1701</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1702" href="#t1702">1702</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1703" href="#t1703">1703</a></span><span class="t">                <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">gated_mlp</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1703&#x202F;&#x219B;&#x202F;1704</span><span class="annotate long">line 1703 didn't jump to line 1704</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1704" href="#t1704">1704</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_gate"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1703" href="#t1703">1703</a></span><span class="t">                <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">gated_mlp</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1704" href="#t1704">1704</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_gate"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1705" href="#t1705">1705</a></span><span class="t">                        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_gate"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1706" href="#t1706">1706</a></span><span class="t">                        <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln2.w"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1707" href="#t1707">1707</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1708" href="#t1708">1708</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
     <p class="run"><span class="n"><a id="t1709" href="#t1709">1709</a></span><span class="t">                <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln2.w"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1710" href="#t1710">1710</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1711" href="#t1711">1711</a></span><span class="t">                <span class="key">if</span> <span class="nam">center_weights</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1711&#x202F;&#x219B;&#x202F;1719</span><span class="annotate long">line 1711 didn't jump to line 1719, because the condition on line 1711 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t1711" href="#t1711">1711</a></span><span class="t">                <span class="key">if</span> <span class="nam">center_weights</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1712" href="#t1712">1712</a></span><span class="t">                    <span class="com"># Center the weights that read in from the LayerNormPre</span>&nbsp;</span><span class="r"></span></p>
     <p class="run"><span class="n"><a id="t1713" href="#t1713">1713</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_in"</span><span class="op">]</span> <span class="op">-=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">reduce</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1714" href="#t1714">1714</a></span><span class="t">                        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_in"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
@@ -1841,7 +1841,7 @@ <h2>
     <p class="run"><span class="n"><a id="t1757" href="#t1757">1757</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"unembed.W_U"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"unembed.W_U"</span><span class="op">]</span> <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"ln_final.w"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
     <p class="run"><span class="n"><a id="t1758" href="#t1758">1758</a></span><span class="t">        <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"ln_final.w"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1759" href="#t1759">1759</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1760" href="#t1760">1760</a></span><span class="t">        <span class="key">if</span> <span class="nam">center_weights</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1760&#x202F;&#x219B;&#x202F;1766</span><span class="annotate long">line 1760 didn't jump to line 1766, because the condition on line 1760 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t1760" href="#t1760">1760</a></span><span class="t">        <span class="key">if</span> <span class="nam">center_weights</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1761" href="#t1761">1761</a></span><span class="t">            <span class="com"># Center the weights that read in from the LayerNormPre</span>&nbsp;</span><span class="r"></span></p>
     <p class="run"><span class="n"><a id="t1762" href="#t1762">1762</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"unembed.W_U"</span><span class="op">]</span> <span class="op">-=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">reduce</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1763" href="#t1763">1763</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"unembed.W_U"</span><span class="op">]</span><span class="op">,</span> <span class="str">"d_model d_vocab -> 1 d_vocab"</span><span class="op">,</span> <span class="str">"mean"</span>&nbsp;</span><span class="r"></span></p>
@@ -1909,11 +1909,11 @@ <h2>
     <p class="pln"><span class="n"><a id="t1825" href="#t1825">1825</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
     <p class="run"><span class="n"><a id="t1826" href="#t1826">1826</a></span><span class="t">        <span class="key">for</span> <span class="nam">layer</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1827" href="#t1827">1827</a></span><span class="t">            <span class="com"># shape [head_index, d_head]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1828" href="#t1828">1828</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_key_value_heads</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1828&#x202F;&#x219B;&#x202F;1831</span><span class="annotate long">line 1828 didn't jump to line 1831, because the condition on line 1828 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t1828" href="#t1828">1828</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_key_value_heads</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
     <p class="run"><span class="n"><a id="t1829" href="#t1829">1829</a></span><span class="t">                <span class="nam">b_V</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn.b_V"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1830" href="#t1830">1830</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1831" href="#t1831">1831</a></span><span class="t">                <span class="nam">b_V</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn._b_V"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1832" href="#t1832">1832</a></span><span class="t">                <span class="nam">b_V</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">repeat_interleave</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1831" href="#t1831">1831</a></span><span class="t">                <span class="nam">b_V</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn._b_V"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1832" href="#t1832">1832</a></span><span class="t">                <span class="nam">b_V</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">repeat_interleave</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1833" href="#t1833">1833</a></span><span class="t">                    <span class="nam">b_V</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">,</span> <span class="nam">repeats</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_heads</span> <span class="op">//</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_key_value_heads</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1834" href="#t1834">1834</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1835" href="#t1835">1835</a></span><span class="t">            <span class="com"># [head_index, d_head, d_model]</span>&nbsp;</span><span class="r"></span></p>
@@ -1923,10 +1923,10 @@ <h2>
     <p class="run"><span class="n"><a id="t1839" href="#t1839">1839</a></span><span class="t">            <span class="nam">folded_b_O</span> <span class="op">=</span> <span class="nam">b_O_original</span> <span class="op">+</span> <span class="op">(</span><span class="nam">b_V</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span> <span class="op">*</span> <span class="nam">W_O</span><span class="op">)</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span><span class="op">[</span><span class="num">0</span><span class="op">,</span> <span class="num">1</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1840" href="#t1840">1840</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
     <p class="run"><span class="n"><a id="t1841" href="#t1841">1841</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn.b_O"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">folded_b_O</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1842" href="#t1842">1842</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_key_value_heads</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1842&#x202F;&#x219B;&#x202F;1845</span><span class="annotate long">line 1842 didn't jump to line 1845, because the condition on line 1842 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t1842" href="#t1842">1842</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_key_value_heads</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
     <p class="run"><span class="n"><a id="t1843" href="#t1843">1843</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn.b_V"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros_like</span><span class="op">(</span><span class="nam">b_V</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1844" href="#t1844">1844</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1845" href="#t1845">1845</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn._b_V"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros_like</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1845" href="#t1845">1845</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn._b_V"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros_like</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1846" href="#t1846">1846</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn._b_V"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1847" href="#t1847">1847</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="run"><span class="n"><a id="t1848" href="#t1848">1848</a></span><span class="t">        <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
@@ -2053,535 +2053,538 @@ <h2>
     <p class="pln"><span class="n"><a id="t1969" href="#t1969">1969</a></span><span class="t">        <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1970" href="#t1970">1970</a></span><span class="t"><span class="str">        Toggles whether to allow editing of inputs to each attention head.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1971" href="#t1971">1971</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1972" href="#t1972">1972</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">use_attn_in</span> <span class="op">=</span> <span class="nam">use_attn_in</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1973" href="#t1973">1973</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1974" href="#t1974">1974</a></span><span class="t">    <span class="key">def</span> <span class="nam">set_ungroup_grouped_query_attention</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">ungroup_grouped_query_attention</span><span class="op">:</span> <span class="nam">bool</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1975" href="#t1975">1975</a></span><span class="t">        <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1976" href="#t1976">1976</a></span><span class="t"><span class="str">        Toggles whether to ungroup the grouped key and value heads in models with grouped query attention (GQA).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1977" href="#t1977">1977</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1978" href="#t1978">1978</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">ungroup_grouped_query_attention</span> <span class="op">=</span> <span class="nam">ungroup_grouped_query_attention</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1979" href="#t1979">1979</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1980" href="#t1980">1980</a></span><span class="t">    <span class="key">def</span> <span class="nam">process_weights_</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1981" href="#t1981">1981</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1982" href="#t1982">1982</a></span><span class="t">        <span class="nam">fold_ln</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1983" href="#t1983">1983</a></span><span class="t">        <span class="nam">center_writing_weights</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1984" href="#t1984">1984</a></span><span class="t">        <span class="nam">center_unembed</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1985" href="#t1985">1985</a></span><span class="t">        <span class="nam">refactor_factored_attn_matrices</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1986" href="#t1986">1986</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1987" href="#t1987">1987</a></span><span class="t">        <span class="str">"""Wrapper around `load_and_process_state_dict`.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1988" href="#t1988">1988</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1989" href="#t1989">1989</a></span><span class="t"><span class="str">        Wrapper around load_and_process_state_dict to allow for in-place processing of the weights.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1990" href="#t1990">1990</a></span><span class="t"><span class="str">        This is useful if using HookedTransformer for training, if we then want to analyse a cleaner</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1991" href="#t1991">1991</a></span><span class="t"><span class="str">        version of the same model.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1992" href="#t1992">1992</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1993" href="#t1993">1993</a></span><span class="t">        <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">state_dict</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1994" href="#t1994">1994</a></span><span class="t">        <span class="key">if</span> <span class="nam">fold_ln</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">num_experts</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">num_experts</span> <span class="op">></span> <span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1994&#x202F;&#x219B;&#x202F;1997</span><span class="annotate long">line 1994 didn't jump to line 1997, because the condition on line 1994 was never true</span></span></p>
-    <p class="pln"><span class="n"><a id="t1995" href="#t1995">1995</a></span><span class="t">            <span class="com"># If we're using MoE, we don't fold the layer norm weights, so we don't need to do any preprocessing</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1996" href="#t1996">1996</a></span><span class="t">            <span class="com"># A warning is already issued in `load_and_process_state_dict`</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1997" href="#t1997">1997</a></span><span class="t">            <span class="key">pass</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1998" href="#t1998">1998</a></span><span class="t">        <span class="key">elif</span> <span class="nam">fold_ln</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">==</span> <span class="str">"LN"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1998&#x202F;&#x219B;&#x202F;2009</span><span class="annotate long">line 1998 didn't jump to line 2009, because the condition on line 1998 was never false</span></span></p>
-    <p class="pln"><span class="n"><a id="t1999" href="#t1999">1999</a></span><span class="t">            <span class="com"># If we're folding the LN into the weights, we need to replace all the layernorm layers</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2000" href="#t2000">2000</a></span><span class="t">            <span class="com"># with LayerNormPres, which do not have learnable parameters. This is somewhat hacky,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2001" href="#t2001">2001</a></span><span class="t">            <span class="com"># but it's the easiest way to do it.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2002" href="#t2002">2002</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">=</span> <span class="str">"LNPre"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2003" href="#t2003">2003</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">ln_final</span> <span class="op">=</span> <span class="nam">LayerNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2004" href="#t2004">2004</a></span><span class="t">            <span class="key">for</span> <span class="nam">layer</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2005" href="#t2005">2005</a></span><span class="t">                <span class="nam">layer</span><span class="op">.</span><span class="nam">ln1</span> <span class="op">=</span> <span class="nam">LayerNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2006" href="#t2006">2006</a></span><span class="t">                <span class="nam">layer</span><span class="op">.</span><span class="nam">ln2</span> <span class="op">=</span> <span class="nam">LayerNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2007" href="#t2007">2007</a></span><span class="t">                <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">is_layer_norm_activation</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2007&#x202F;&#x219B;&#x202F;2008</span><span class="annotate long">line 2007 didn't jump to line 2008, because the condition on line 2007 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2008" href="#t2008">2008</a></span><span class="t">                    <span class="nam">layer</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">ln</span> <span class="op">=</span> <span class="nam">LayerNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2009" href="#t2009">2009</a></span><span class="t">        <span class="key">elif</span> <span class="nam">fold_ln</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">==</span> <span class="str">"RMS"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2010" href="#t2010">2010</a></span><span class="t">            <span class="com"># We do the same for RMSNorm if used</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2011" href="#t2011">2011</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">=</span> <span class="str">"RMSPre"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2012" href="#t2012">2012</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">ln_final</span> <span class="op">=</span> <span class="nam">RMSNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2013" href="#t2013">2013</a></span><span class="t">            <span class="key">for</span> <span class="nam">layer</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2014" href="#t2014">2014</a></span><span class="t">                <span class="nam">layer</span><span class="op">.</span><span class="nam">ln1</span> <span class="op">=</span> <span class="nam">RMSNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2015" href="#t2015">2015</a></span><span class="t">                <span class="nam">layer</span><span class="op">.</span><span class="nam">ln2</span> <span class="op">=</span> <span class="nam">RMSNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2016" href="#t2016">2016</a></span><span class="t">                <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">is_layer_norm_activation</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2017" href="#t2017">2017</a></span><span class="t">                    <span class="nam">layer</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">ln</span> <span class="op">=</span> <span class="nam">RMSNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2018" href="#t2018">2018</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2019" href="#t2019">2019</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">load_and_process_state_dict</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2020" href="#t2020">2020</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2021" href="#t2021">2021</a></span><span class="t">            <span class="nam">fold_ln</span><span class="op">=</span><span class="nam">fold_ln</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2022" href="#t2022">2022</a></span><span class="t">            <span class="nam">center_writing_weights</span><span class="op">=</span><span class="nam">center_writing_weights</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2023" href="#t2023">2023</a></span><span class="t">            <span class="nam">center_unembed</span><span class="op">=</span><span class="nam">center_unembed</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2024" href="#t2024">2024</a></span><span class="t">            <span class="nam">refactor_factored_attn_matrices</span><span class="op">=</span><span class="nam">refactor_factored_attn_matrices</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2025" href="#t2025">2025</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2026" href="#t2026">2026</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2027" href="#t2027">2027</a></span><span class="t">    <span class="op">@</span><span class="nam">torch</span><span class="op">.</span><span class="nam">inference_mode</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2028" href="#t2028">2028</a></span><span class="t">    <span class="key">def</span> <span class="nam">generate</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2029" href="#t2029">2029</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2030" href="#t2030">2030</a></span><span class="t">        <span class="nam">input</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="str">""</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2031" href="#t2031">2031</a></span><span class="t">        <span class="nam">max_new_tokens</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">10</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2032" href="#t2032">2032</a></span><span class="t">        <span class="nam">stop_at_eos</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2033" href="#t2033">2033</a></span><span class="t">        <span class="nam">eos_token_id</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2034" href="#t2034">2034</a></span><span class="t">        <span class="nam">do_sample</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2035" href="#t2035">2035</a></span><span class="t">        <span class="nam">top_k</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2036" href="#t2036">2036</a></span><span class="t">        <span class="nam">top_p</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">float</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2037" href="#t2037">2037</a></span><span class="t">        <span class="nam">temperature</span><span class="op">:</span> <span class="nam">float</span> <span class="op">=</span> <span class="num">1.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2038" href="#t2038">2038</a></span><span class="t">        <span class="nam">freq_penalty</span><span class="op">:</span> <span class="nam">float</span> <span class="op">=</span> <span class="num">0.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2039" href="#t2039">2039</a></span><span class="t">        <span class="nam">use_past_kv_cache</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2040" href="#t2040">2040</a></span><span class="t">        <span class="nam">prepend_bos</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">bool</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2041" href="#t2041">2041</a></span><span class="t">        <span class="nam">padding_side</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Literal</span><span class="op">[</span><span class="str">"left"</span><span class="op">,</span> <span class="str">"right"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2042" href="#t2042">2042</a></span><span class="t">        <span class="nam">return_type</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span> <span class="op">=</span> <span class="str">"input"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2043" href="#t2043">2043</a></span><span class="t">        <span class="nam">verbose</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2044" href="#t2044">2044</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos_plus_new_tokens"</span><span class="op">]</span><span class="op">,</span> <span class="nam">str</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2045" href="#t2045">2045</a></span><span class="t">        <span class="str">"""Sample Tokens from the Model.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2046" href="#t2046">2046</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2047" href="#t2047">2047</a></span><span class="t"><span class="str">        Sample tokens from the model until the model outputs eos_token or max_new_tokens is reached.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2048" href="#t2048">2048</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2049" href="#t2049">2049</a></span><span class="t"><span class="str">        To avoid fiddling with ragged tensors, if we input a batch of text and some sequences finish</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2050" href="#t2050">2050</a></span><span class="t"><span class="str">        (by producing an EOT token), we keep running the model on the entire batch, but throw away</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2051" href="#t2051">2051</a></span><span class="t"><span class="str">        the output for a finished sequence and just keep adding EOTs to pad.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2052" href="#t2052">2052</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2053" href="#t2053">2053</a></span><span class="t"><span class="str">        This supports entering a single string, but not a list of strings - if the strings don't</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2054" href="#t2054">2054</a></span><span class="t"><span class="str">        tokenize to exactly the same length, this gets messy. If that functionality is needed,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2055" href="#t2055">2055</a></span><span class="t"><span class="str">        convert them to a batch of tokens and input that instead.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2056" href="#t2056">2056</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2057" href="#t2057">2057</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2058" href="#t2058">2058</a></span><span class="t"><span class="str">            input (Union[str, Int[torch.Tensor, "batch pos"])]): Either a batch of tokens ([batch,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2059" href="#t2059">2059</a></span><span class="t"><span class="str">                pos]) or a text string (this will be converted to a batch of tokens with batch size</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2060" href="#t2060">2060</a></span><span class="t"><span class="str">                1).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2061" href="#t2061">2061</a></span><span class="t"><span class="str">            max_new_tokens (int): Maximum number of tokens to generate.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2062" href="#t2062">2062</a></span><span class="t"><span class="str">            stop_at_eos (bool): If True, stop generating tokens when the model outputs eos_token.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2063" href="#t2063">2063</a></span><span class="t"><span class="str">            eos_token_id (Optional[Union[int, Sequence]]): The token ID to use for end</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2064" href="#t2064">2064</a></span><span class="t"><span class="str">                of sentence. If None, use the tokenizer's eos_token_id - required if using</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2065" href="#t2065">2065</a></span><span class="t"><span class="str">                stop_at_eos. It's also possible to provide a list of token IDs (not just the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2066" href="#t2066">2066</a></span><span class="t"><span class="str">                eos_token_id), in which case the generation will stop when any of them are output</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2067" href="#t2067">2067</a></span><span class="t"><span class="str">                (useful e.g. for stable_lm).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2068" href="#t2068">2068</a></span><span class="t"><span class="str">            do_sample (bool): If True, sample from the model's output distribution. Otherwise, use</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2069" href="#t2069">2069</a></span><span class="t"><span class="str">                greedy search (take the max logit each time).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2070" href="#t2070">2070</a></span><span class="t"><span class="str">            top_k (int): Number of tokens to sample from. If None, sample from all tokens.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2071" href="#t2071">2071</a></span><span class="t"><span class="str">            top_p (float): Probability mass to sample from. If 1.0, sample from all tokens. If &lt;1.0,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2072" href="#t2072">2072</a></span><span class="t"><span class="str">                we take the top tokens with cumulative probability >= top_p.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2073" href="#t2073">2073</a></span><span class="t"><span class="str">            temperature (float): Temperature for sampling. Higher values will make the model more</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2074" href="#t2074">2074</a></span><span class="t"><span class="str">                random (limit of temp -> 0 is just taking the top token, limit of temp -> inf is</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2075" href="#t2075">2075</a></span><span class="t"><span class="str">                sampling from a uniform distribution).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2076" href="#t2076">2076</a></span><span class="t"><span class="str">            freq_penalty (float): Frequency penalty for sampling - how much to penalise previous</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2077" href="#t2077">2077</a></span><span class="t"><span class="str">                tokens. Higher values will make the model more random.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2078" href="#t2078">2078</a></span><span class="t"><span class="str">            use_past_kv_cache (bool): If True, create and use cache to speed up generation.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2079" href="#t2079">2079</a></span><span class="t"><span class="str">            prepend_bos (bool, optional): Overrides self.cfg.default_prepend_bos. Whether to prepend</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2080" href="#t2080">2080</a></span><span class="t"><span class="str">                the BOS token to the input (applicable when input is a string). Defaults to None,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2081" href="#t2081">2081</a></span><span class="t"><span class="str">                implying usage of self.cfg.default_prepend_bos (default is True unless specified</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2082" href="#t2082">2082</a></span><span class="t"><span class="str">                otherwise). Pass True or False to override the default.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2083" href="#t2083">2083</a></span><span class="t"><span class="str">            padding_side (Union[Literal["left", "right"], None], optional): Overrides</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2084" href="#t2084">2084</a></span><span class="t"><span class="str">                self.tokenizer.padding_side. Specifies which side to pad when tokenizing multiple</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2085" href="#t2085">2085</a></span><span class="t"><span class="str">                strings of different lengths.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2086" href="#t2086">2086</a></span><span class="t"><span class="str">            return_type (Optional[str]): The type of the output to return - either a string (str),</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2087" href="#t2087">2087</a></span><span class="t"><span class="str">                a tensor of tokens (tensor) or whatever the format of the input was (input).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2088" href="#t2088">2088</a></span><span class="t"><span class="str">            verbose (bool): If True, show tqdm progress bars for generation.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2089" href="#t2089">2089</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2090" href="#t2090">2090</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2091" href="#t2091">2091</a></span><span class="t"><span class="str">            outputs (torch.Tensor): [batch, pos + max_new_tokens], generated sequence of new tokens</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2092" href="#t2092">2092</a></span><span class="t"><span class="str">                (by default returns same type as input).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2093" href="#t2093">2093</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2094" href="#t2094">2094</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2095" href="#t2095">2095</a></span><span class="t">        <span class="key">with</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">LocallyOverridenDefaults</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2096" href="#t2096">2096</a></span><span class="t">            <span class="nam">self</span><span class="op">,</span> <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span> <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2097" href="#t2097">2097</a></span><span class="t">        <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2098" href="#t2098">2098</a></span><span class="t">            <span class="key">if</span> <span class="nam">type</span><span class="op">(</span><span class="nam">input</span><span class="op">)</span> <span class="op">==</span> <span class="nam">str</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2098&#x202F;&#x219B;&#x202F;2105</span><span class="annotate long">line 2098 didn't jump to line 2105, because the condition on line 2098 was never false</span></span></p>
-    <p class="pln"><span class="n"><a id="t2099" href="#t2099">2099</a></span><span class="t">                <span class="com"># If text, convert to tokens (batch_size=1)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2100" href="#t2100">2100</a></span><span class="t">                <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2101" href="#t2101">2101</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2102" href="#t2102">2102</a></span><span class="t">                <span class="op">)</span><span class="op">,</span> <span class="str">"Must provide a tokenizer if passing a string to the model"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2103" href="#t2103">2103</a></span><span class="t">                <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">to_tokens</span><span class="op">(</span><span class="nam">input</span><span class="op">,</span> <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span> <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2104" href="#t2104">2104</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2105" href="#t2105">2105</a></span><span class="t">                <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">input</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2106" href="#t2106">2106</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2107" href="#t2107">2107</a></span><span class="t">            <span class="key">if</span> <span class="nam">return_type</span> <span class="op">==</span> <span class="str">"input"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2107&#x202F;&#x219B;&#x202F;2113</span><span class="annotate long">line 2107 didn't jump to line 2113, because the condition on line 2107 was never false</span></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2108" href="#t2108">2108</a></span><span class="t">                <span class="key">if</span> <span class="nam">type</span><span class="op">(</span><span class="nam">input</span><span class="op">)</span> <span class="op">==</span> <span class="nam">str</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2108&#x202F;&#x219B;&#x202F;2111</span><span class="annotate long">line 2108 didn't jump to line 2111, because the condition on line 2108 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t2109" href="#t2109">2109</a></span><span class="t">                    <span class="nam">return_type</span> <span class="op">=</span> <span class="str">"str"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2110" href="#t2110">2110</a></span><span class="t">                <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2111" href="#t2111">2111</a></span><span class="t">                    <span class="nam">return_type</span> <span class="op">=</span> <span class="str">"tensor"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2112" href="#t2112">2112</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2113" href="#t2113">2113</a></span><span class="t">            <span class="key">assert</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2114" href="#t2114">2114</a></span><span class="t">            <span class="nam">batch_size</span><span class="op">,</span> <span class="nam">ctx_length</span> <span class="op">=</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">shape</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2115" href="#t2115">2115</a></span><span class="t">            <span class="nam">device</span> <span class="op">=</span> <span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2116" href="#t2116">2116</a></span><span class="t">            <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">device</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2117" href="#t2117">2117</a></span><span class="t">            <span class="key">if</span> <span class="nam">use_past_kv_cache</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2117&#x202F;&#x219B;&#x202F;2122</span><span class="annotate long">line 2117 didn't jump to line 2122, because the condition on line 2117 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t2118" href="#t2118">2118</a></span><span class="t">                <span class="nam">past_kv_cache</span> <span class="op">=</span> <span class="nam">HookedTransformerKeyValueCache</span><span class="op">.</span><span class="nam">init_cache</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2119" href="#t2119">2119</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">,</span> <span class="nam">batch_size</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2120" href="#t2120">2120</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2121" href="#t2121">2121</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2122" href="#t2122">2122</a></span><span class="t">                <span class="nam">past_kv_cache</span> <span class="op">=</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2123" href="#t2123">2123</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2124" href="#t2124">2124</a></span><span class="t">            <span class="nam">stop_tokens</span><span class="op">:</span> <span class="nam">List</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2125" href="#t2125">2125</a></span><span class="t">            <span class="nam">eos_token_for_padding</span> <span class="op">=</span> <span class="num">0</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2126" href="#t2126">2126</a></span><span class="t">            <span class="key">assert</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2127" href="#t2127">2127</a></span><span class="t">            <span class="key">if</span> <span class="nam">stop_at_eos</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2127&#x202F;&#x219B;&#x202F;2149</span><span class="annotate long">line 2127 didn't jump to line 2149, because the condition on line 2127 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t2128" href="#t2128">2128</a></span><span class="t">                <span class="nam">tokenizer_has_eos_token</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2129" href="#t2129">2129</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">eos_token_id</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2130" href="#t2130">2130</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2131" href="#t2131">2131</a></span><span class="t">                <span class="key">if</span> <span class="nam">eos_token_id</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2131&#x202F;&#x219B;&#x202F;2138</span><span class="annotate long">line 2131 didn't jump to line 2138, because the condition on line 2131 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t2132" href="#t2132">2132</a></span><span class="t">                    <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2133" href="#t2133">2133</a></span><span class="t">                        <span class="nam">tokenizer_has_eos_token</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2134" href="#t2134">2134</a></span><span class="t">                    <span class="op">)</span><span class="op">,</span> <span class="str">"Must pass a eos_token_id if stop_at_eos is True and tokenizer is None or has no eos_token_id"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2135" href="#t2135">2135</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2136" href="#t2136">2136</a></span><span class="t">                    <span class="nam">eos_token_id</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">eos_token_id</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2137" href="#t2137">2137</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2138" href="#t2138">2138</a></span><span class="t">                <span class="key">if</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">eos_token_id</span><span class="op">,</span> <span class="nam">int</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2138&#x202F;&#x219B;&#x202F;2143</span><span class="annotate long">line 2138 didn't jump to line 2143, because the condition on line 2138 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t2139" href="#t2139">2139</a></span><span class="t">                    <span class="nam">stop_tokens</span> <span class="op">=</span> <span class="op">[</span><span class="nam">eos_token_id</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2140" href="#t2140">2140</a></span><span class="t">                    <span class="nam">eos_token_for_padding</span> <span class="op">=</span> <span class="nam">eos_token_id</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2141" href="#t2141">2141</a></span><span class="t">                <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2142" href="#t2142">2142</a></span><span class="t">                    <span class="com"># eos_token_id is a Sequence (e.g. list or tuple)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2143" href="#t2143">2143</a></span><span class="t">                    <span class="nam">stop_tokens</span> <span class="op">=</span> <span class="nam">eos_token_id</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2144" href="#t2144">2144</a></span><span class="t">                    <span class="nam">eos_token_for_padding</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2145" href="#t2145">2145</a></span><span class="t">                        <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">eos_token_id</span> <span class="key">if</span> <span class="nam">tokenizer_has_eos_token</span> <span class="key">else</span> <span class="nam">eos_token_id</span><span class="op">[</span><span class="num">0</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2146" href="#t2146">2146</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2147" href="#t2147">2147</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2148" href="#t2148">2148</a></span><span class="t">            <span class="com"># An array to track which sequences in the batch have finished.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2149" href="#t2149">2149</a></span><span class="t">            <span class="nam">finished_sequences</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros</span><span class="op">(</span><span class="nam">batch_size</span><span class="op">,</span> <span class="nam">dtype</span><span class="op">=</span><span class="nam">torch</span><span class="op">.</span><span class="nam">bool</span><span class="op">,</span> <span class="nam">device</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1972" href="#t1972">1972</a></span><span class="t">        <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1973" href="#t1973">1973</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_key_value_heads</span> <span class="key">is</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1974" href="#t1974">1974</a></span><span class="t">        <span class="op">)</span><span class="op">,</span> <span class="str">"Can't use attn_in with GroupedQueryAttention, please use split_qkv_input instead"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1975" href="#t1975">1975</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">use_attn_in</span> <span class="op">=</span> <span class="nam">use_attn_in</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1976" href="#t1976">1976</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1977" href="#t1977">1977</a></span><span class="t">    <span class="key">def</span> <span class="nam">set_ungroup_grouped_query_attention</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">ungroup_grouped_query_attention</span><span class="op">:</span> <span class="nam">bool</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1978" href="#t1978">1978</a></span><span class="t">        <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1979" href="#t1979">1979</a></span><span class="t"><span class="str">        Toggles whether to ungroup the grouped key and value heads in models with grouped query attention (GQA).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1980" href="#t1980">1980</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1981" href="#t1981">1981</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">ungroup_grouped_query_attention</span> <span class="op">=</span> <span class="nam">ungroup_grouped_query_attention</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1982" href="#t1982">1982</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1983" href="#t1983">1983</a></span><span class="t">    <span class="key">def</span> <span class="nam">process_weights_</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1984" href="#t1984">1984</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1985" href="#t1985">1985</a></span><span class="t">        <span class="nam">fold_ln</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1986" href="#t1986">1986</a></span><span class="t">        <span class="nam">center_writing_weights</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1987" href="#t1987">1987</a></span><span class="t">        <span class="nam">center_unembed</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1988" href="#t1988">1988</a></span><span class="t">        <span class="nam">refactor_factored_attn_matrices</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1989" href="#t1989">1989</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1990" href="#t1990">1990</a></span><span class="t">        <span class="str">"""Wrapper around `load_and_process_state_dict`.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1991" href="#t1991">1991</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1992" href="#t1992">1992</a></span><span class="t"><span class="str">        Wrapper around load_and_process_state_dict to allow for in-place processing of the weights.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1993" href="#t1993">1993</a></span><span class="t"><span class="str">        This is useful if using HookedTransformer for training, if we then want to analyse a cleaner</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1994" href="#t1994">1994</a></span><span class="t"><span class="str">        version of the same model.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1995" href="#t1995">1995</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1996" href="#t1996">1996</a></span><span class="t">        <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">state_dict</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1997" href="#t1997">1997</a></span><span class="t">        <span class="key">if</span> <span class="nam">fold_ln</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">num_experts</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">num_experts</span> <span class="op">></span> <span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1997&#x202F;&#x219B;&#x202F;2000</span><span class="annotate long">line 1997 didn't jump to line 2000, because the condition on line 1997 was never true</span></span></p>
+    <p class="pln"><span class="n"><a id="t1998" href="#t1998">1998</a></span><span class="t">            <span class="com"># If we're using MoE, we don't fold the layer norm weights, so we don't need to do any preprocessing</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1999" href="#t1999">1999</a></span><span class="t">            <span class="com"># A warning is already issued in `load_and_process_state_dict`</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2000" href="#t2000">2000</a></span><span class="t">            <span class="key">pass</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2001" href="#t2001">2001</a></span><span class="t">        <span class="key">elif</span> <span class="nam">fold_ln</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">==</span> <span class="str">"LN"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2001&#x202F;&#x219B;&#x202F;2012</span><span class="annotate long">line 2001 didn't jump to line 2012, because the condition on line 2001 was never false</span></span></p>
+    <p class="pln"><span class="n"><a id="t2002" href="#t2002">2002</a></span><span class="t">            <span class="com"># If we're folding the LN into the weights, we need to replace all the layernorm layers</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2003" href="#t2003">2003</a></span><span class="t">            <span class="com"># with LayerNormPres, which do not have learnable parameters. This is somewhat hacky,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2004" href="#t2004">2004</a></span><span class="t">            <span class="com"># but it's the easiest way to do it.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2005" href="#t2005">2005</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">=</span> <span class="str">"LNPre"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2006" href="#t2006">2006</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">ln_final</span> <span class="op">=</span> <span class="nam">LayerNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2007" href="#t2007">2007</a></span><span class="t">            <span class="key">for</span> <span class="nam">layer</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2008" href="#t2008">2008</a></span><span class="t">                <span class="nam">layer</span><span class="op">.</span><span class="nam">ln1</span> <span class="op">=</span> <span class="nam">LayerNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2009" href="#t2009">2009</a></span><span class="t">                <span class="nam">layer</span><span class="op">.</span><span class="nam">ln2</span> <span class="op">=</span> <span class="nam">LayerNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2010" href="#t2010">2010</a></span><span class="t">                <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">is_layer_norm_activation</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2010&#x202F;&#x219B;&#x202F;2011</span><span class="annotate long">line 2010 didn't jump to line 2011, because the condition on line 2010 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2011" href="#t2011">2011</a></span><span class="t">                    <span class="nam">layer</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">ln</span> <span class="op">=</span> <span class="nam">LayerNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2012" href="#t2012">2012</a></span><span class="t">        <span class="key">elif</span> <span class="nam">fold_ln</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">==</span> <span class="str">"RMS"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2013" href="#t2013">2013</a></span><span class="t">            <span class="com"># We do the same for RMSNorm if used</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2014" href="#t2014">2014</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">=</span> <span class="str">"RMSPre"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2015" href="#t2015">2015</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">ln_final</span> <span class="op">=</span> <span class="nam">RMSNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2016" href="#t2016">2016</a></span><span class="t">            <span class="key">for</span> <span class="nam">layer</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2017" href="#t2017">2017</a></span><span class="t">                <span class="nam">layer</span><span class="op">.</span><span class="nam">ln1</span> <span class="op">=</span> <span class="nam">RMSNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2018" href="#t2018">2018</a></span><span class="t">                <span class="nam">layer</span><span class="op">.</span><span class="nam">ln2</span> <span class="op">=</span> <span class="nam">RMSNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2019" href="#t2019">2019</a></span><span class="t">                <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">is_layer_norm_activation</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2020" href="#t2020">2020</a></span><span class="t">                    <span class="nam">layer</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">ln</span> <span class="op">=</span> <span class="nam">RMSNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2021" href="#t2021">2021</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2022" href="#t2022">2022</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">load_and_process_state_dict</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2023" href="#t2023">2023</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2024" href="#t2024">2024</a></span><span class="t">            <span class="nam">fold_ln</span><span class="op">=</span><span class="nam">fold_ln</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2025" href="#t2025">2025</a></span><span class="t">            <span class="nam">center_writing_weights</span><span class="op">=</span><span class="nam">center_writing_weights</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2026" href="#t2026">2026</a></span><span class="t">            <span class="nam">center_unembed</span><span class="op">=</span><span class="nam">center_unembed</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2027" href="#t2027">2027</a></span><span class="t">            <span class="nam">refactor_factored_attn_matrices</span><span class="op">=</span><span class="nam">refactor_factored_attn_matrices</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2028" href="#t2028">2028</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2029" href="#t2029">2029</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2030" href="#t2030">2030</a></span><span class="t">    <span class="op">@</span><span class="nam">torch</span><span class="op">.</span><span class="nam">inference_mode</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2031" href="#t2031">2031</a></span><span class="t">    <span class="key">def</span> <span class="nam">generate</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2032" href="#t2032">2032</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2033" href="#t2033">2033</a></span><span class="t">        <span class="nam">input</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="str">""</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2034" href="#t2034">2034</a></span><span class="t">        <span class="nam">max_new_tokens</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">10</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2035" href="#t2035">2035</a></span><span class="t">        <span class="nam">stop_at_eos</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2036" href="#t2036">2036</a></span><span class="t">        <span class="nam">eos_token_id</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2037" href="#t2037">2037</a></span><span class="t">        <span class="nam">do_sample</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2038" href="#t2038">2038</a></span><span class="t">        <span class="nam">top_k</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2039" href="#t2039">2039</a></span><span class="t">        <span class="nam">top_p</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">float</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2040" href="#t2040">2040</a></span><span class="t">        <span class="nam">temperature</span><span class="op">:</span> <span class="nam">float</span> <span class="op">=</span> <span class="num">1.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2041" href="#t2041">2041</a></span><span class="t">        <span class="nam">freq_penalty</span><span class="op">:</span> <span class="nam">float</span> <span class="op">=</span> <span class="num">0.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2042" href="#t2042">2042</a></span><span class="t">        <span class="nam">use_past_kv_cache</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2043" href="#t2043">2043</a></span><span class="t">        <span class="nam">prepend_bos</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">bool</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2044" href="#t2044">2044</a></span><span class="t">        <span class="nam">padding_side</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Literal</span><span class="op">[</span><span class="str">"left"</span><span class="op">,</span> <span class="str">"right"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2045" href="#t2045">2045</a></span><span class="t">        <span class="nam">return_type</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span> <span class="op">=</span> <span class="str">"input"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2046" href="#t2046">2046</a></span><span class="t">        <span class="nam">verbose</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2047" href="#t2047">2047</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos_plus_new_tokens"</span><span class="op">]</span><span class="op">,</span> <span class="nam">str</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2048" href="#t2048">2048</a></span><span class="t">        <span class="str">"""Sample Tokens from the Model.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2049" href="#t2049">2049</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2050" href="#t2050">2050</a></span><span class="t"><span class="str">        Sample tokens from the model until the model outputs eos_token or max_new_tokens is reached.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2051" href="#t2051">2051</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2052" href="#t2052">2052</a></span><span class="t"><span class="str">        To avoid fiddling with ragged tensors, if we input a batch of text and some sequences finish</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2053" href="#t2053">2053</a></span><span class="t"><span class="str">        (by producing an EOT token), we keep running the model on the entire batch, but throw away</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2054" href="#t2054">2054</a></span><span class="t"><span class="str">        the output for a finished sequence and just keep adding EOTs to pad.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2055" href="#t2055">2055</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2056" href="#t2056">2056</a></span><span class="t"><span class="str">        This supports entering a single string, but not a list of strings - if the strings don't</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2057" href="#t2057">2057</a></span><span class="t"><span class="str">        tokenize to exactly the same length, this gets messy. If that functionality is needed,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2058" href="#t2058">2058</a></span><span class="t"><span class="str">        convert them to a batch of tokens and input that instead.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2059" href="#t2059">2059</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2060" href="#t2060">2060</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2061" href="#t2061">2061</a></span><span class="t"><span class="str">            input (Union[str, Int[torch.Tensor, "batch pos"])]): Either a batch of tokens ([batch,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2062" href="#t2062">2062</a></span><span class="t"><span class="str">                pos]) or a text string (this will be converted to a batch of tokens with batch size</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2063" href="#t2063">2063</a></span><span class="t"><span class="str">                1).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2064" href="#t2064">2064</a></span><span class="t"><span class="str">            max_new_tokens (int): Maximum number of tokens to generate.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2065" href="#t2065">2065</a></span><span class="t"><span class="str">            stop_at_eos (bool): If True, stop generating tokens when the model outputs eos_token.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2066" href="#t2066">2066</a></span><span class="t"><span class="str">            eos_token_id (Optional[Union[int, Sequence]]): The token ID to use for end</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2067" href="#t2067">2067</a></span><span class="t"><span class="str">                of sentence. If None, use the tokenizer's eos_token_id - required if using</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2068" href="#t2068">2068</a></span><span class="t"><span class="str">                stop_at_eos. It's also possible to provide a list of token IDs (not just the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2069" href="#t2069">2069</a></span><span class="t"><span class="str">                eos_token_id), in which case the generation will stop when any of them are output</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2070" href="#t2070">2070</a></span><span class="t"><span class="str">                (useful e.g. for stable_lm).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2071" href="#t2071">2071</a></span><span class="t"><span class="str">            do_sample (bool): If True, sample from the model's output distribution. Otherwise, use</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2072" href="#t2072">2072</a></span><span class="t"><span class="str">                greedy search (take the max logit each time).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2073" href="#t2073">2073</a></span><span class="t"><span class="str">            top_k (int): Number of tokens to sample from. If None, sample from all tokens.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2074" href="#t2074">2074</a></span><span class="t"><span class="str">            top_p (float): Probability mass to sample from. If 1.0, sample from all tokens. If &lt;1.0,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2075" href="#t2075">2075</a></span><span class="t"><span class="str">                we take the top tokens with cumulative probability >= top_p.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2076" href="#t2076">2076</a></span><span class="t"><span class="str">            temperature (float): Temperature for sampling. Higher values will make the model more</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2077" href="#t2077">2077</a></span><span class="t"><span class="str">                random (limit of temp -> 0 is just taking the top token, limit of temp -> inf is</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2078" href="#t2078">2078</a></span><span class="t"><span class="str">                sampling from a uniform distribution).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2079" href="#t2079">2079</a></span><span class="t"><span class="str">            freq_penalty (float): Frequency penalty for sampling - how much to penalise previous</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2080" href="#t2080">2080</a></span><span class="t"><span class="str">                tokens. Higher values will make the model more random.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2081" href="#t2081">2081</a></span><span class="t"><span class="str">            use_past_kv_cache (bool): If True, create and use cache to speed up generation.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2082" href="#t2082">2082</a></span><span class="t"><span class="str">            prepend_bos (bool, optional): Overrides self.cfg.default_prepend_bos. Whether to prepend</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2083" href="#t2083">2083</a></span><span class="t"><span class="str">                the BOS token to the input (applicable when input is a string). Defaults to None,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2084" href="#t2084">2084</a></span><span class="t"><span class="str">                implying usage of self.cfg.default_prepend_bos (default is True unless specified</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2085" href="#t2085">2085</a></span><span class="t"><span class="str">                otherwise). Pass True or False to override the default.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2086" href="#t2086">2086</a></span><span class="t"><span class="str">            padding_side (Union[Literal["left", "right"], None], optional): Overrides</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2087" href="#t2087">2087</a></span><span class="t"><span class="str">                self.tokenizer.padding_side. Specifies which side to pad when tokenizing multiple</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2088" href="#t2088">2088</a></span><span class="t"><span class="str">                strings of different lengths.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2089" href="#t2089">2089</a></span><span class="t"><span class="str">            return_type (Optional[str]): The type of the output to return - either a string (str),</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2090" href="#t2090">2090</a></span><span class="t"><span class="str">                a tensor of tokens (tensor) or whatever the format of the input was (input).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2091" href="#t2091">2091</a></span><span class="t"><span class="str">            verbose (bool): If True, show tqdm progress bars for generation.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2092" href="#t2092">2092</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2093" href="#t2093">2093</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2094" href="#t2094">2094</a></span><span class="t"><span class="str">            outputs (torch.Tensor): [batch, pos + max_new_tokens], generated sequence of new tokens</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2095" href="#t2095">2095</a></span><span class="t"><span class="str">                (by default returns same type as input).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2096" href="#t2096">2096</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2097" href="#t2097">2097</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2098" href="#t2098">2098</a></span><span class="t">        <span class="key">with</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">LocallyOverridenDefaults</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2099" href="#t2099">2099</a></span><span class="t">            <span class="nam">self</span><span class="op">,</span> <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span> <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2100" href="#t2100">2100</a></span><span class="t">        <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2101" href="#t2101">2101</a></span><span class="t">            <span class="key">if</span> <span class="nam">type</span><span class="op">(</span><span class="nam">input</span><span class="op">)</span> <span class="op">==</span> <span class="nam">str</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2101&#x202F;&#x219B;&#x202F;2108</span><span class="annotate long">line 2101 didn't jump to line 2108, because the condition on line 2101 was never false</span></span></p>
+    <p class="pln"><span class="n"><a id="t2102" href="#t2102">2102</a></span><span class="t">                <span class="com"># If text, convert to tokens (batch_size=1)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2103" href="#t2103">2103</a></span><span class="t">                <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2104" href="#t2104">2104</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2105" href="#t2105">2105</a></span><span class="t">                <span class="op">)</span><span class="op">,</span> <span class="str">"Must provide a tokenizer if passing a string to the model"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2106" href="#t2106">2106</a></span><span class="t">                <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">to_tokens</span><span class="op">(</span><span class="nam">input</span><span class="op">,</span> <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span> <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2107" href="#t2107">2107</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2108" href="#t2108">2108</a></span><span class="t">                <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">input</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2109" href="#t2109">2109</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2110" href="#t2110">2110</a></span><span class="t">            <span class="key">if</span> <span class="nam">return_type</span> <span class="op">==</span> <span class="str">"input"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2110&#x202F;&#x219B;&#x202F;2116</span><span class="annotate long">line 2110 didn't jump to line 2116, because the condition on line 2110 was never false</span></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2111" href="#t2111">2111</a></span><span class="t">                <span class="key">if</span> <span class="nam">type</span><span class="op">(</span><span class="nam">input</span><span class="op">)</span> <span class="op">==</span> <span class="nam">str</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2111&#x202F;&#x219B;&#x202F;2114</span><span class="annotate long">line 2111 didn't jump to line 2114, because the condition on line 2111 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t2112" href="#t2112">2112</a></span><span class="t">                    <span class="nam">return_type</span> <span class="op">=</span> <span class="str">"str"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2113" href="#t2113">2113</a></span><span class="t">                <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2114" href="#t2114">2114</a></span><span class="t">                    <span class="nam">return_type</span> <span class="op">=</span> <span class="str">"tensor"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2115" href="#t2115">2115</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2116" href="#t2116">2116</a></span><span class="t">            <span class="key">assert</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2117" href="#t2117">2117</a></span><span class="t">            <span class="nam">batch_size</span><span class="op">,</span> <span class="nam">ctx_length</span> <span class="op">=</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">shape</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2118" href="#t2118">2118</a></span><span class="t">            <span class="nam">device</span> <span class="op">=</span> <span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2119" href="#t2119">2119</a></span><span class="t">            <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">device</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2120" href="#t2120">2120</a></span><span class="t">            <span class="key">if</span> <span class="nam">use_past_kv_cache</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2120&#x202F;&#x219B;&#x202F;2125</span><span class="annotate long">line 2120 didn't jump to line 2125, because the condition on line 2120 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t2121" href="#t2121">2121</a></span><span class="t">                <span class="nam">past_kv_cache</span> <span class="op">=</span> <span class="nam">HookedTransformerKeyValueCache</span><span class="op">.</span><span class="nam">init_cache</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2122" href="#t2122">2122</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">,</span> <span class="nam">batch_size</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2123" href="#t2123">2123</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2124" href="#t2124">2124</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2125" href="#t2125">2125</a></span><span class="t">                <span class="nam">past_kv_cache</span> <span class="op">=</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2126" href="#t2126">2126</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2127" href="#t2127">2127</a></span><span class="t">            <span class="nam">stop_tokens</span><span class="op">:</span> <span class="nam">List</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2128" href="#t2128">2128</a></span><span class="t">            <span class="nam">eos_token_for_padding</span> <span class="op">=</span> <span class="num">0</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2129" href="#t2129">2129</a></span><span class="t">            <span class="key">assert</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2130" href="#t2130">2130</a></span><span class="t">            <span class="key">if</span> <span class="nam">stop_at_eos</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2130&#x202F;&#x219B;&#x202F;2152</span><span class="annotate long">line 2130 didn't jump to line 2152, because the condition on line 2130 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t2131" href="#t2131">2131</a></span><span class="t">                <span class="nam">tokenizer_has_eos_token</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2132" href="#t2132">2132</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">eos_token_id</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2133" href="#t2133">2133</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2134" href="#t2134">2134</a></span><span class="t">                <span class="key">if</span> <span class="nam">eos_token_id</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2134&#x202F;&#x219B;&#x202F;2141</span><span class="annotate long">line 2134 didn't jump to line 2141, because the condition on line 2134 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t2135" href="#t2135">2135</a></span><span class="t">                    <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2136" href="#t2136">2136</a></span><span class="t">                        <span class="nam">tokenizer_has_eos_token</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2137" href="#t2137">2137</a></span><span class="t">                    <span class="op">)</span><span class="op">,</span> <span class="str">"Must pass a eos_token_id if stop_at_eos is True and tokenizer is None or has no eos_token_id"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2138" href="#t2138">2138</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2139" href="#t2139">2139</a></span><span class="t">                    <span class="nam">eos_token_id</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">eos_token_id</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2140" href="#t2140">2140</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2141" href="#t2141">2141</a></span><span class="t">                <span class="key">if</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">eos_token_id</span><span class="op">,</span> <span class="nam">int</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2141&#x202F;&#x219B;&#x202F;2146</span><span class="annotate long">line 2141 didn't jump to line 2146, because the condition on line 2141 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t2142" href="#t2142">2142</a></span><span class="t">                    <span class="nam">stop_tokens</span> <span class="op">=</span> <span class="op">[</span><span class="nam">eos_token_id</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2143" href="#t2143">2143</a></span><span class="t">                    <span class="nam">eos_token_for_padding</span> <span class="op">=</span> <span class="nam">eos_token_id</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2144" href="#t2144">2144</a></span><span class="t">                <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2145" href="#t2145">2145</a></span><span class="t">                    <span class="com"># eos_token_id is a Sequence (e.g. list or tuple)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2146" href="#t2146">2146</a></span><span class="t">                    <span class="nam">stop_tokens</span> <span class="op">=</span> <span class="nam">eos_token_id</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2147" href="#t2147">2147</a></span><span class="t">                    <span class="nam">eos_token_for_padding</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2148" href="#t2148">2148</a></span><span class="t">                        <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">eos_token_id</span> <span class="key">if</span> <span class="nam">tokenizer_has_eos_token</span> <span class="key">else</span> <span class="nam">eos_token_id</span><span class="op">[</span><span class="num">0</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2149" href="#t2149">2149</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t2150" href="#t2150">2150</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2151" href="#t2151">2151</a></span><span class="t">            <span class="com"># Currently nothing in HookedTransformer changes with eval, but this is here in case</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2152" href="#t2152">2152</a></span><span class="t">            <span class="com"># that changes in the future.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2153" href="#t2153">2153</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">eval</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2154" href="#t2154">2154</a></span><span class="t">            <span class="key">for</span> <span class="nam">index</span> <span class="key">in</span> <span class="nam">tqdm</span><span class="op">.</span><span class="nam">tqdm</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="nam">max_new_tokens</span><span class="op">)</span><span class="op">,</span> <span class="nam">disable</span><span class="op">=</span><span class="key">not</span> <span class="nam">verbose</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2155" href="#t2155">2155</a></span><span class="t">                <span class="com"># While generating, we keep generating logits, throw away all but the final logits,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2156" href="#t2156">2156</a></span><span class="t">                <span class="com"># and then use those logits to sample from the distribution We keep adding the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2157" href="#t2157">2157</a></span><span class="t">                <span class="com"># sampled tokens to the end of tokens.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2158" href="#t2158">2158</a></span><span class="t">                <span class="key">if</span> <span class="nam">use_past_kv_cache</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2158&#x202F;&#x219B;&#x202F;2179</span><span class="annotate long">line 2158 didn't jump to line 2179, because the condition on line 2158 was never false</span></span></p>
-    <p class="pln"><span class="n"><a id="t2159" href="#t2159">2159</a></span><span class="t">                    <span class="com"># We just take the final tokens, as a [batch, 1] tensor</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2160" href="#t2160">2160</a></span><span class="t">                    <span class="key">if</span> <span class="nam">index</span> <span class="op">></span> <span class="num">0</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2161" href="#t2161">2161</a></span><span class="t">                        <span class="nam">logits</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">forward</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2162" href="#t2162">2162</a></span><span class="t">                            <span class="nam">tokens</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="op">-</span><span class="num">1</span><span class="op">:</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2163" href="#t2163">2163</a></span><span class="t">                            <span class="nam">return_type</span><span class="op">=</span><span class="str">"logits"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2164" href="#t2164">2164</a></span><span class="t">                            <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2165" href="#t2165">2165</a></span><span class="t">                            <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2166" href="#t2166">2166</a></span><span class="t">                            <span class="nam">past_kv_cache</span><span class="op">=</span><span class="nam">past_kv_cache</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2167" href="#t2167">2167</a></span><span class="t">                        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2168" href="#t2168">2168</a></span><span class="t">                    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2169" href="#t2169">2169</a></span><span class="t">                        <span class="nam">logits</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">forward</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2170" href="#t2170">2170</a></span><span class="t">                            <span class="nam">tokens</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2171" href="#t2171">2171</a></span><span class="t">                            <span class="nam">return_type</span><span class="op">=</span><span class="str">"logits"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2172" href="#t2172">2172</a></span><span class="t">                            <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2173" href="#t2173">2173</a></span><span class="t">                            <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2174" href="#t2174">2174</a></span><span class="t">                            <span class="nam">past_kv_cache</span><span class="op">=</span><span class="nam">past_kv_cache</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2175" href="#t2175">2175</a></span><span class="t">                        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2176" href="#t2176">2176</a></span><span class="t">                <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2177" href="#t2177">2177</a></span><span class="t">                    <span class="com"># We input the entire sequence, as a [batch, pos] tensor, since we aren't using</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2178" href="#t2178">2178</a></span><span class="t">                    <span class="com"># the cache.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2179" href="#t2179">2179</a></span><span class="t">                    <span class="nam">logits</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">forward</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2180" href="#t2180">2180</a></span><span class="t">                        <span class="nam">tokens</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2181" href="#t2181">2181</a></span><span class="t">                        <span class="nam">return_type</span><span class="op">=</span><span class="str">"logits"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2182" href="#t2182">2182</a></span><span class="t">                        <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2183" href="#t2183">2183</a></span><span class="t">                        <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2184" href="#t2184">2184</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2185" href="#t2185">2185</a></span><span class="t">                <span class="nam">final_logits</span> <span class="op">=</span> <span class="nam">logits</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2186" href="#t2186">2186</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2187" href="#t2187">2187</a></span><span class="t">                <span class="key">if</span> <span class="nam">do_sample</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2187&#x202F;&#x219B;&#x202F;2188</span><span class="annotate long">line 2187 didn't jump to line 2188, because the condition on line 2187 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2188" href="#t2188">2188</a></span><span class="t">                    <span class="nam">sampled_tokens</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">sample_logits</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2189" href="#t2189">2189</a></span><span class="t">                        <span class="nam">final_logits</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2190" href="#t2190">2190</a></span><span class="t">                        <span class="nam">top_k</span><span class="op">=</span><span class="nam">top_k</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2191" href="#t2191">2191</a></span><span class="t">                        <span class="nam">top_p</span><span class="op">=</span><span class="nam">top_p</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2192" href="#t2192">2192</a></span><span class="t">                        <span class="nam">temperature</span><span class="op">=</span><span class="nam">temperature</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2193" href="#t2193">2193</a></span><span class="t">                        <span class="nam">freq_penalty</span><span class="op">=</span><span class="nam">freq_penalty</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2194" href="#t2194">2194</a></span><span class="t">                        <span class="nam">tokens</span><span class="op">=</span><span class="nam">tokens</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2195" href="#t2195">2195</a></span><span class="t">                    <span class="op">)</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2196" href="#t2196">2196</a></span><span class="t">                <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2197" href="#t2197">2197</a></span><span class="t">                    <span class="nam">sampled_tokens</span> <span class="op">=</span> <span class="nam">final_logits</span><span class="op">.</span><span class="nam">argmax</span><span class="op">(</span><span class="op">-</span><span class="num">1</span><span class="op">)</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2198" href="#t2198">2198</a></span><span class="t">                        <span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2199" href="#t2199">2199</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2200" href="#t2200">2200</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2201" href="#t2201">2201</a></span><span class="t">                <span class="key">if</span> <span class="nam">stop_at_eos</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2201&#x202F;&#x219B;&#x202F;2213</span><span class="annotate long">line 2201 didn't jump to line 2213, because the condition on line 2201 was never false</span></span></p>
-    <p class="pln"><span class="n"><a id="t2202" href="#t2202">2202</a></span><span class="t">                    <span class="com"># For all unfinished sequences, add on the next token. If a sequence was</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2203" href="#t2203">2203</a></span><span class="t">                    <span class="com"># finished, throw away the generated token and add eos_token_for_padding</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2204" href="#t2204">2204</a></span><span class="t">                    <span class="com"># instead.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2205" href="#t2205">2205</a></span><span class="t">                    <span class="nam">sampled_tokens</span><span class="op">[</span><span class="nam">finished_sequences</span><span class="op">]</span> <span class="op">=</span> <span class="nam">eos_token_for_padding</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2206" href="#t2206">2206</a></span><span class="t">                    <span class="nam">finished_sequences</span><span class="op">.</span><span class="nam">logical_or_</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2207" href="#t2207">2207</a></span><span class="t">                        <span class="nam">torch</span><span class="op">.</span><span class="nam">isin</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2208" href="#t2208">2208</a></span><span class="t">                            <span class="nam">sampled_tokens</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2209" href="#t2209">2209</a></span><span class="t">                            <span class="nam">torch</span><span class="op">.</span><span class="nam">tensor</span><span class="op">(</span><span class="nam">stop_tokens</span><span class="op">)</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2210" href="#t2210">2210</a></span><span class="t">                        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2211" href="#t2211">2211</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2212" href="#t2212">2212</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2213" href="#t2213">2213</a></span><span class="t">                <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cat</span><span class="op">(</span><span class="op">[</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">sampled_tokens</span><span class="op">.</span><span class="nam">unsqueeze</span><span class="op">(</span><span class="op">-</span><span class="num">1</span><span class="op">)</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">1</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2214" href="#t2214">2214</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2215" href="#t2215">2215</a></span><span class="t">                <span class="key">if</span> <span class="nam">stop_at_eos</span> <span class="key">and</span> <span class="nam">finished_sequences</span><span class="op">.</span><span class="nam">all</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2215&#x202F;&#x219B;&#x202F;2216</span><span class="annotate long">line 2215 didn't jump to line 2216, because the condition on line 2215 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2216" href="#t2216">2216</a></span><span class="t">                    <span class="key">break</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2151" href="#t2151">2151</a></span><span class="t">            <span class="com"># An array to track which sequences in the batch have finished.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2152" href="#t2152">2152</a></span><span class="t">            <span class="nam">finished_sequences</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros</span><span class="op">(</span><span class="nam">batch_size</span><span class="op">,</span> <span class="nam">dtype</span><span class="op">=</span><span class="nam">torch</span><span class="op">.</span><span class="nam">bool</span><span class="op">,</span> <span class="nam">device</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2153" href="#t2153">2153</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2154" href="#t2154">2154</a></span><span class="t">            <span class="com"># Currently nothing in HookedTransformer changes with eval, but this is here in case</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2155" href="#t2155">2155</a></span><span class="t">            <span class="com"># that changes in the future.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2156" href="#t2156">2156</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">eval</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2157" href="#t2157">2157</a></span><span class="t">            <span class="key">for</span> <span class="nam">index</span> <span class="key">in</span> <span class="nam">tqdm</span><span class="op">.</span><span class="nam">tqdm</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="nam">max_new_tokens</span><span class="op">)</span><span class="op">,</span> <span class="nam">disable</span><span class="op">=</span><span class="key">not</span> <span class="nam">verbose</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2158" href="#t2158">2158</a></span><span class="t">                <span class="com"># While generating, we keep generating logits, throw away all but the final logits,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2159" href="#t2159">2159</a></span><span class="t">                <span class="com"># and then use those logits to sample from the distribution We keep adding the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2160" href="#t2160">2160</a></span><span class="t">                <span class="com"># sampled tokens to the end of tokens.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2161" href="#t2161">2161</a></span><span class="t">                <span class="key">if</span> <span class="nam">use_past_kv_cache</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2161&#x202F;&#x219B;&#x202F;2182</span><span class="annotate long">line 2161 didn't jump to line 2182, because the condition on line 2161 was never false</span></span></p>
+    <p class="pln"><span class="n"><a id="t2162" href="#t2162">2162</a></span><span class="t">                    <span class="com"># We just take the final tokens, as a [batch, 1] tensor</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2163" href="#t2163">2163</a></span><span class="t">                    <span class="key">if</span> <span class="nam">index</span> <span class="op">></span> <span class="num">0</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2164" href="#t2164">2164</a></span><span class="t">                        <span class="nam">logits</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">forward</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2165" href="#t2165">2165</a></span><span class="t">                            <span class="nam">tokens</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="op">-</span><span class="num">1</span><span class="op">:</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2166" href="#t2166">2166</a></span><span class="t">                            <span class="nam">return_type</span><span class="op">=</span><span class="str">"logits"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2167" href="#t2167">2167</a></span><span class="t">                            <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2168" href="#t2168">2168</a></span><span class="t">                            <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2169" href="#t2169">2169</a></span><span class="t">                            <span class="nam">past_kv_cache</span><span class="op">=</span><span class="nam">past_kv_cache</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2170" href="#t2170">2170</a></span><span class="t">                        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2171" href="#t2171">2171</a></span><span class="t">                    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2172" href="#t2172">2172</a></span><span class="t">                        <span class="nam">logits</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">forward</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2173" href="#t2173">2173</a></span><span class="t">                            <span class="nam">tokens</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2174" href="#t2174">2174</a></span><span class="t">                            <span class="nam">return_type</span><span class="op">=</span><span class="str">"logits"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2175" href="#t2175">2175</a></span><span class="t">                            <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2176" href="#t2176">2176</a></span><span class="t">                            <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2177" href="#t2177">2177</a></span><span class="t">                            <span class="nam">past_kv_cache</span><span class="op">=</span><span class="nam">past_kv_cache</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2178" href="#t2178">2178</a></span><span class="t">                        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2179" href="#t2179">2179</a></span><span class="t">                <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2180" href="#t2180">2180</a></span><span class="t">                    <span class="com"># We input the entire sequence, as a [batch, pos] tensor, since we aren't using</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2181" href="#t2181">2181</a></span><span class="t">                    <span class="com"># the cache.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2182" href="#t2182">2182</a></span><span class="t">                    <span class="nam">logits</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">forward</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2183" href="#t2183">2183</a></span><span class="t">                        <span class="nam">tokens</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2184" href="#t2184">2184</a></span><span class="t">                        <span class="nam">return_type</span><span class="op">=</span><span class="str">"logits"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2185" href="#t2185">2185</a></span><span class="t">                        <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2186" href="#t2186">2186</a></span><span class="t">                        <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2187" href="#t2187">2187</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2188" href="#t2188">2188</a></span><span class="t">                <span class="nam">final_logits</span> <span class="op">=</span> <span class="nam">logits</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2189" href="#t2189">2189</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2190" href="#t2190">2190</a></span><span class="t">                <span class="key">if</span> <span class="nam">do_sample</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2190&#x202F;&#x219B;&#x202F;2191</span><span class="annotate long">line 2190 didn't jump to line 2191, because the condition on line 2190 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2191" href="#t2191">2191</a></span><span class="t">                    <span class="nam">sampled_tokens</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">sample_logits</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2192" href="#t2192">2192</a></span><span class="t">                        <span class="nam">final_logits</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2193" href="#t2193">2193</a></span><span class="t">                        <span class="nam">top_k</span><span class="op">=</span><span class="nam">top_k</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2194" href="#t2194">2194</a></span><span class="t">                        <span class="nam">top_p</span><span class="op">=</span><span class="nam">top_p</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2195" href="#t2195">2195</a></span><span class="t">                        <span class="nam">temperature</span><span class="op">=</span><span class="nam">temperature</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2196" href="#t2196">2196</a></span><span class="t">                        <span class="nam">freq_penalty</span><span class="op">=</span><span class="nam">freq_penalty</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2197" href="#t2197">2197</a></span><span class="t">                        <span class="nam">tokens</span><span class="op">=</span><span class="nam">tokens</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2198" href="#t2198">2198</a></span><span class="t">                    <span class="op">)</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2199" href="#t2199">2199</a></span><span class="t">                <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2200" href="#t2200">2200</a></span><span class="t">                    <span class="nam">sampled_tokens</span> <span class="op">=</span> <span class="nam">final_logits</span><span class="op">.</span><span class="nam">argmax</span><span class="op">(</span><span class="op">-</span><span class="num">1</span><span class="op">)</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2201" href="#t2201">2201</a></span><span class="t">                        <span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2202" href="#t2202">2202</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2203" href="#t2203">2203</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2204" href="#t2204">2204</a></span><span class="t">                <span class="key">if</span> <span class="nam">stop_at_eos</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2204&#x202F;&#x219B;&#x202F;2216</span><span class="annotate long">line 2204 didn't jump to line 2216, because the condition on line 2204 was never false</span></span></p>
+    <p class="pln"><span class="n"><a id="t2205" href="#t2205">2205</a></span><span class="t">                    <span class="com"># For all unfinished sequences, add on the next token. If a sequence was</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2206" href="#t2206">2206</a></span><span class="t">                    <span class="com"># finished, throw away the generated token and add eos_token_for_padding</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2207" href="#t2207">2207</a></span><span class="t">                    <span class="com"># instead.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2208" href="#t2208">2208</a></span><span class="t">                    <span class="nam">sampled_tokens</span><span class="op">[</span><span class="nam">finished_sequences</span><span class="op">]</span> <span class="op">=</span> <span class="nam">eos_token_for_padding</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2209" href="#t2209">2209</a></span><span class="t">                    <span class="nam">finished_sequences</span><span class="op">.</span><span class="nam">logical_or_</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2210" href="#t2210">2210</a></span><span class="t">                        <span class="nam">torch</span><span class="op">.</span><span class="nam">isin</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2211" href="#t2211">2211</a></span><span class="t">                            <span class="nam">sampled_tokens</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2212" href="#t2212">2212</a></span><span class="t">                            <span class="nam">torch</span><span class="op">.</span><span class="nam">tensor</span><span class="op">(</span><span class="nam">stop_tokens</span><span class="op">)</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2213" href="#t2213">2213</a></span><span class="t">                        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2214" href="#t2214">2214</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2215" href="#t2215">2215</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2216" href="#t2216">2216</a></span><span class="t">                <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cat</span><span class="op">(</span><span class="op">[</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">sampled_tokens</span><span class="op">.</span><span class="nam">unsqueeze</span><span class="op">(</span><span class="op">-</span><span class="num">1</span><span class="op">)</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">1</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t2217" href="#t2217">2217</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2218" href="#t2218">2218</a></span><span class="t">            <span class="key">if</span> <span class="nam">return_type</span> <span class="op">==</span> <span class="str">"str"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2218&#x202F;&#x219B;&#x202F;2226</span><span class="annotate long">line 2218 didn't jump to line 2226, because the condition on line 2218 was never false</span></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2219" href="#t2219">2219</a></span><span class="t">                <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">default_prepend_bos</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2219&#x202F;&#x219B;&#x202F;2221</span><span class="annotate long">line 2219 didn't jump to line 2221, because the condition on line 2219 was never true</span></span></p>
-    <p class="pln"><span class="n"><a id="t2220" href="#t2220">2220</a></span><span class="t">                    <span class="com"># If we prepended a BOS token, remove it when returning output.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2221" href="#t2221">2221</a></span><span class="t">                    <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">decode</span><span class="op">(</span><span class="nam">tokens</span><span class="op">[</span><span class="num">0</span><span class="op">,</span> <span class="num">1</span><span class="op">:</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2222" href="#t2222">2222</a></span><span class="t">                <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2223" href="#t2223">2223</a></span><span class="t">                    <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">decode</span><span class="op">(</span><span class="nam">tokens</span><span class="op">[</span><span class="num">0</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2224" href="#t2224">2224</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2225" href="#t2225">2225</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2226" href="#t2226">2226</a></span><span class="t">                <span class="key">return</span> <span class="nam">tokens</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2218" href="#t2218">2218</a></span><span class="t">                <span class="key">if</span> <span class="nam">stop_at_eos</span> <span class="key">and</span> <span class="nam">finished_sequences</span><span class="op">.</span><span class="nam">all</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2218&#x202F;&#x219B;&#x202F;2219</span><span class="annotate long">line 2218 didn't jump to line 2219, because the condition on line 2218 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2219" href="#t2219">2219</a></span><span class="t">                    <span class="key">break</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2220" href="#t2220">2220</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2221" href="#t2221">2221</a></span><span class="t">            <span class="key">if</span> <span class="nam">return_type</span> <span class="op">==</span> <span class="str">"str"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2221&#x202F;&#x219B;&#x202F;2229</span><span class="annotate long">line 2221 didn't jump to line 2229, because the condition on line 2221 was never false</span></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2222" href="#t2222">2222</a></span><span class="t">                <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">default_prepend_bos</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2222&#x202F;&#x219B;&#x202F;2224</span><span class="annotate long">line 2222 didn't jump to line 2224, because the condition on line 2222 was never true</span></span></p>
+    <p class="pln"><span class="n"><a id="t2223" href="#t2223">2223</a></span><span class="t">                    <span class="com"># If we prepended a BOS token, remove it when returning output.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2224" href="#t2224">2224</a></span><span class="t">                    <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">decode</span><span class="op">(</span><span class="nam">tokens</span><span class="op">[</span><span class="num">0</span><span class="op">,</span> <span class="num">1</span><span class="op">:</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2225" href="#t2225">2225</a></span><span class="t">                <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2226" href="#t2226">2226</a></span><span class="t">                    <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">decode</span><span class="op">(</span><span class="nam">tokens</span><span class="op">[</span><span class="num">0</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t2227" href="#t2227">2227</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2228" href="#t2228">2228</a></span><span class="t">    <span class="com"># Give access to all weights as properties.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2229" href="#t2229">2229</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2230" href="#t2230">2230</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_U</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"d_model d_vocab"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2231" href="#t2231">2231</a></span><span class="t">        <span class="str">"""Convenience to get the unembedding matrix.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2232" href="#t2232">2232</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2233" href="#t2233">2233</a></span><span class="t"><span class="str">        I.e. the linear map from the final residual stream to the output logits).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2234" href="#t2234">2234</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2235" href="#t2235">2235</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">unembed</span><span class="op">.</span><span class="nam">W_U</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2236" href="#t2236">2236</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2237" href="#t2237">2237</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2238" href="#t2238">2238</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_U</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"d_vocab"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2239" href="#t2239">2239</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">unembed</span><span class="op">.</span><span class="nam">b_U</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2240" href="#t2240">2240</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2241" href="#t2241">2241</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2242" href="#t2242">2242</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_E</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"d_vocab d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2243" href="#t2243">2243</a></span><span class="t">        <span class="str">"""Convenience to get the embedding matrix."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2244" href="#t2244">2244</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">embed</span><span class="op">.</span><span class="nam">W_E</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2245" href="#t2245">2245</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2246" href="#t2246">2246</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2247" href="#t2247">2247</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_pos</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_ctx d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2248" href="#t2248">2248</a></span><span class="t">        <span class="str">"""Convenience function to get the positional embedding.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2249" href="#t2249">2249</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2250" href="#t2250">2250</a></span><span class="t"><span class="str">        Only works on models with absolute positional embeddings!</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2251" href="#t2251">2251</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2252" href="#t2252">2252</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">pos_embed</span><span class="op">.</span><span class="nam">W_pos</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2253" href="#t2253">2253</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2254" href="#t2254">2254</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2255" href="#t2255">2255</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_E_pos</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"d_vocab+n_ctx d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2256" href="#t2256">2256</a></span><span class="t">        <span class="str">"""Concatenated W_E and W_pos.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2257" href="#t2257">2257</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2258" href="#t2258">2258</a></span><span class="t"><span class="str">        Used as a full (overcomplete) basis of the input space, useful for full QK and full OV</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2259" href="#t2259">2259</a></span><span class="t"><span class="str">        circuits.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2260" href="#t2260">2260</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2261" href="#t2261">2261</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cat</span><span class="op">(</span><span class="op">[</span><span class="nam">self</span><span class="op">.</span><span class="nam">W_E</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">W_pos</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2262" href="#t2262">2262</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2263" href="#t2263">2263</a></span><span class="t">    <span class="com"># Layer-specific weights are stacked into one massive tensor and given as properties for</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2264" href="#t2264">2264</a></span><span class="t">    <span class="com"># convenience and a cache is used to avoid repeated computation. Often a useful convenience when</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2265" href="#t2265">2265</a></span><span class="t">    <span class="com"># we want to do analysis on weights across all layers. If GPU memory is a bottleneck, don't use</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2266" href="#t2266">2266</a></span><span class="t">    <span class="com"># these properties!</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2267" href="#t2267">2267</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2268" href="#t2268">2268</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2269" href="#t2269">2269</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_K</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_model d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2270" href="#t2270">2270</a></span><span class="t">        <span class="str">"""Stack the key weights across all layers."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2271" href="#t2271">2271</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">W_K</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2271&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2271&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2271 didn't run the list comprehension on line 2271, 2) line 2271 didn't return from function 'W_K', because the return on line 2271 wasn't executed</span></span></p>
-    <p class="pln"><span class="n"><a id="t2272" href="#t2272">2272</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2273" href="#t2273">2273</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2274" href="#t2274">2274</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_Q</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_model d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2275" href="#t2275">2275</a></span><span class="t">        <span class="str">"""Stack the query weights across all layers."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2276" href="#t2276">2276</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">W_Q</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2276&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2276&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2276 didn't run the list comprehension on line 2276, 2) line 2276 didn't return from function 'W_Q', because the return on line 2276 wasn't executed</span></span></p>
-    <p class="pln"><span class="n"><a id="t2277" href="#t2277">2277</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2278" href="#t2278">2278</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2279" href="#t2279">2279</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_V</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_model d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2280" href="#t2280">2280</a></span><span class="t">        <span class="str">"""Stack the value weights across all layers."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2281" href="#t2281">2281</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">W_V</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2281&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2281&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2281 didn't run the list comprehension on line 2281, 2) line 2281 didn't return from function 'W_V', because the return on line 2281 wasn't executed</span></span></p>
-    <p class="pln"><span class="n"><a id="t2282" href="#t2282">2282</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2283" href="#t2283">2283</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2284" href="#t2284">2284</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_O</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_head d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2285" href="#t2285">2285</a></span><span class="t">        <span class="str">"""Stack the attn output weights across all layers."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2286" href="#t2286">2286</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">W_O</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2286&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2286&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2286 didn't run the list comprehension on line 2286, 2) line 2286 didn't return from function 'W_O', because the return on line 2286 wasn't executed</span></span></p>
-    <p class="pln"><span class="n"><a id="t2287" href="#t2287">2287</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2288" href="#t2288">2288</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2289" href="#t2289">2289</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_in</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers d_model d_mlp"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2290" href="#t2290">2290</a></span><span class="t">        <span class="str">"""Stack the MLP input weights across all layers."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2291" href="#t2291">2291</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">W_in</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2291&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2291&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2291 didn't run the list comprehension on line 2291, 2) line 2291 didn't return from function 'W_in', because the return on line 2291 wasn't executed</span></span></p>
-    <p class="pln"><span class="n"><a id="t2292" href="#t2292">2292</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2293" href="#t2293">2293</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2294" href="#t2294">2294</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_gate</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers d_model d_mlp"</span><span class="op">]</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2295" href="#t2295">2295</a></span><span class="t">        <span class="str">"""Stack the MLP gate weights across all layers.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2296" href="#t2296">2296</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2297" href="#t2297">2297</a></span><span class="t"><span class="str">        Only works for models with gated MLPs.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2298" href="#t2298">2298</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2299" href="#t2299">2299</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">gated_mlp</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2300" href="#t2300">2300</a></span><span class="t">            <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">W_gate</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2301" href="#t2301">2301</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2302" href="#t2302">2302</a></span><span class="t">            <span class="key">return</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2303" href="#t2303">2303</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2304" href="#t2304">2304</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2305" href="#t2305">2305</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_out</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers d_mlp d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2306" href="#t2306">2306</a></span><span class="t">        <span class="str">"""Stack the MLP output weights across all layers."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2307" href="#t2307">2307</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">W_out</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2307&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2307&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2307 didn't run the list comprehension on line 2307, 2) line 2307 didn't return from function 'W_out', because the return on line 2307 wasn't executed</span></span></p>
-    <p class="pln"><span class="n"><a id="t2308" href="#t2308">2308</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2309" href="#t2309">2309</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2310" href="#t2310">2310</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_K</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2311" href="#t2311">2311</a></span><span class="t">        <span class="str">"""Stack the key biases across all layers."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2312" href="#t2312">2312</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">b_K</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2312&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2312&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2312 didn't run the list comprehension on line 2312, 2) line 2312 didn't return from function 'b_K', because the return on line 2312 wasn't executed</span></span></p>
-    <p class="pln"><span class="n"><a id="t2313" href="#t2313">2313</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2314" href="#t2314">2314</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2315" href="#t2315">2315</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_Q</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2316" href="#t2316">2316</a></span><span class="t">        <span class="str">"""Stack the query biases across all layers."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2317" href="#t2317">2317</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">b_Q</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2317&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2317&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2317 didn't run the list comprehension on line 2317, 2) line 2317 didn't return from function 'b_Q', because the return on line 2317 wasn't executed</span></span></p>
-    <p class="pln"><span class="n"><a id="t2318" href="#t2318">2318</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2319" href="#t2319">2319</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2320" href="#t2320">2320</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_V</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2321" href="#t2321">2321</a></span><span class="t">        <span class="str">"""Stack the value biases across all layers."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2322" href="#t2322">2322</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">b_V</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2322&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2322&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2322 didn't run the list comprehension on line 2322, 2) line 2322 didn't return from function 'b_V', because the return on line 2322 wasn't executed</span></span></p>
-    <p class="pln"><span class="n"><a id="t2323" href="#t2323">2323</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2324" href="#t2324">2324</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2325" href="#t2325">2325</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_O</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2326" href="#t2326">2326</a></span><span class="t">        <span class="str">"""Stack the attn output biases across all layers."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2327" href="#t2327">2327</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">b_O</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2327&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2327&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2327 didn't run the list comprehension on line 2327, 2) line 2327 didn't return from function 'b_O', because the return on line 2327 wasn't executed</span></span></p>
-    <p class="pln"><span class="n"><a id="t2328" href="#t2328">2328</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2329" href="#t2329">2329</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2330" href="#t2330">2330</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_in</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers d_mlp"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2331" href="#t2331">2331</a></span><span class="t">        <span class="str">"""Stack the MLP input biases across all layers."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2332" href="#t2332">2332</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">b_in</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2332&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2332&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2332 didn't run the list comprehension on line 2332, 2) line 2332 didn't return from function 'b_in', because the return on line 2332 wasn't executed</span></span></p>
-    <p class="pln"><span class="n"><a id="t2333" href="#t2333">2333</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2334" href="#t2334">2334</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2335" href="#t2335">2335</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_out</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2336" href="#t2336">2336</a></span><span class="t">        <span class="str">"""Stack the MLP output biases across all layers."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2337" href="#t2337">2337</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">b_out</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2337&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2337&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2337 didn't run the list comprehension on line 2337, 2) line 2337 didn't return from function 'b_out', because the return on line 2337 wasn't executed</span></span></p>
-    <p class="pln"><span class="n"><a id="t2338" href="#t2338">2338</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2339" href="#t2339">2339</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2340" href="#t2340">2340</a></span><span class="t">    <span class="key">def</span> <span class="nam">QK</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2341" href="#t2341">2341</a></span><span class="t">        <span class="key">return</span> <span class="nam">FactoredMatrix</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">W_Q</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">W_K</span><span class="op">.</span><span class="nam">transpose</span><span class="op">(</span><span class="op">-</span><span class="num">2</span><span class="op">,</span> <span class="op">-</span><span class="num">1</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2342" href="#t2342">2342</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2343" href="#t2343">2343</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2344" href="#t2344">2344</a></span><span class="t">    <span class="key">def</span> <span class="nam">OV</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2345" href="#t2345">2345</a></span><span class="t">        <span class="key">return</span> <span class="nam">FactoredMatrix</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">W_V</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">W_O</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2346" href="#t2346">2346</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2347" href="#t2347">2347</a></span><span class="t">    <span class="com"># Various utility functions</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2348" href="#t2348">2348</a></span><span class="t">    <span class="key">def</span> <span class="nam">accumulated_bias</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2349" href="#t2349">2349</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span> <span class="nam">layer</span><span class="op">:</span> <span class="nam">int</span><span class="op">,</span> <span class="nam">mlp_input</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span> <span class="nam">include_mlp_biases</span><span class="op">=</span><span class="key">True</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2350" href="#t2350">2350</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2351" href="#t2351">2351</a></span><span class="t">        <span class="str">"""Accumulated Bias.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2352" href="#t2352">2352</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2353" href="#t2353">2353</a></span><span class="t"><span class="str">        Returns the accumulated bias from all layer outputs (ie the b_Os and b_outs), up to the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2354" href="#t2354">2354</a></span><span class="t"><span class="str">        input of layer L.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2228" href="#t2228">2228</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2229" href="#t2229">2229</a></span><span class="t">                <span class="key">return</span> <span class="nam">tokens</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2230" href="#t2230">2230</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2231" href="#t2231">2231</a></span><span class="t">    <span class="com"># Give access to all weights as properties.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2232" href="#t2232">2232</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2233" href="#t2233">2233</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_U</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"d_model d_vocab"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2234" href="#t2234">2234</a></span><span class="t">        <span class="str">"""Convenience to get the unembedding matrix.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2235" href="#t2235">2235</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2236" href="#t2236">2236</a></span><span class="t"><span class="str">        I.e. the linear map from the final residual stream to the output logits).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2237" href="#t2237">2237</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2238" href="#t2238">2238</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">unembed</span><span class="op">.</span><span class="nam">W_U</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2239" href="#t2239">2239</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2240" href="#t2240">2240</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2241" href="#t2241">2241</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_U</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"d_vocab"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2242" href="#t2242">2242</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">unembed</span><span class="op">.</span><span class="nam">b_U</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2243" href="#t2243">2243</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2244" href="#t2244">2244</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2245" href="#t2245">2245</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_E</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"d_vocab d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2246" href="#t2246">2246</a></span><span class="t">        <span class="str">"""Convenience to get the embedding matrix."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2247" href="#t2247">2247</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">embed</span><span class="op">.</span><span class="nam">W_E</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2248" href="#t2248">2248</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2249" href="#t2249">2249</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2250" href="#t2250">2250</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_pos</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_ctx d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2251" href="#t2251">2251</a></span><span class="t">        <span class="str">"""Convenience function to get the positional embedding.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2252" href="#t2252">2252</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2253" href="#t2253">2253</a></span><span class="t"><span class="str">        Only works on models with absolute positional embeddings!</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2254" href="#t2254">2254</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2255" href="#t2255">2255</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">pos_embed</span><span class="op">.</span><span class="nam">W_pos</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2256" href="#t2256">2256</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2257" href="#t2257">2257</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2258" href="#t2258">2258</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_E_pos</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"d_vocab+n_ctx d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2259" href="#t2259">2259</a></span><span class="t">        <span class="str">"""Concatenated W_E and W_pos.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2260" href="#t2260">2260</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2261" href="#t2261">2261</a></span><span class="t"><span class="str">        Used as a full (overcomplete) basis of the input space, useful for full QK and full OV</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2262" href="#t2262">2262</a></span><span class="t"><span class="str">        circuits.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2263" href="#t2263">2263</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2264" href="#t2264">2264</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cat</span><span class="op">(</span><span class="op">[</span><span class="nam">self</span><span class="op">.</span><span class="nam">W_E</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">W_pos</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2265" href="#t2265">2265</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2266" href="#t2266">2266</a></span><span class="t">    <span class="com"># Layer-specific weights are stacked into one massive tensor and given as properties for</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2267" href="#t2267">2267</a></span><span class="t">    <span class="com"># convenience and a cache is used to avoid repeated computation. Often a useful convenience when</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2268" href="#t2268">2268</a></span><span class="t">    <span class="com"># we want to do analysis on weights across all layers. If GPU memory is a bottleneck, don't use</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2269" href="#t2269">2269</a></span><span class="t">    <span class="com"># these properties!</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2270" href="#t2270">2270</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2271" href="#t2271">2271</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2272" href="#t2272">2272</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_K</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_model d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2273" href="#t2273">2273</a></span><span class="t">        <span class="str">"""Stack the key weights across all layers."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2274" href="#t2274">2274</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">W_K</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2274&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2274&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2274 didn't run the list comprehension on line 2274, 2) line 2274 didn't return from function 'W_K', because the return on line 2274 wasn't executed</span></span></p>
+    <p class="pln"><span class="n"><a id="t2275" href="#t2275">2275</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2276" href="#t2276">2276</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2277" href="#t2277">2277</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_Q</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_model d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2278" href="#t2278">2278</a></span><span class="t">        <span class="str">"""Stack the query weights across all layers."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2279" href="#t2279">2279</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">W_Q</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2279&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2279&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2279 didn't run the list comprehension on line 2279, 2) line 2279 didn't return from function 'W_Q', because the return on line 2279 wasn't executed</span></span></p>
+    <p class="pln"><span class="n"><a id="t2280" href="#t2280">2280</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2281" href="#t2281">2281</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2282" href="#t2282">2282</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_V</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_model d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2283" href="#t2283">2283</a></span><span class="t">        <span class="str">"""Stack the value weights across all layers."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2284" href="#t2284">2284</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">W_V</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2284&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2284&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2284 didn't run the list comprehension on line 2284, 2) line 2284 didn't return from function 'W_V', because the return on line 2284 wasn't executed</span></span></p>
+    <p class="pln"><span class="n"><a id="t2285" href="#t2285">2285</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2286" href="#t2286">2286</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2287" href="#t2287">2287</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_O</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_head d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2288" href="#t2288">2288</a></span><span class="t">        <span class="str">"""Stack the attn output weights across all layers."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2289" href="#t2289">2289</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">W_O</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2289&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2289&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2289 didn't run the list comprehension on line 2289, 2) line 2289 didn't return from function 'W_O', because the return on line 2289 wasn't executed</span></span></p>
+    <p class="pln"><span class="n"><a id="t2290" href="#t2290">2290</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2291" href="#t2291">2291</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2292" href="#t2292">2292</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_in</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers d_model d_mlp"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2293" href="#t2293">2293</a></span><span class="t">        <span class="str">"""Stack the MLP input weights across all layers."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2294" href="#t2294">2294</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">W_in</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2294&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2294&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2294 didn't run the list comprehension on line 2294, 2) line 2294 didn't return from function 'W_in', because the return on line 2294 wasn't executed</span></span></p>
+    <p class="pln"><span class="n"><a id="t2295" href="#t2295">2295</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2296" href="#t2296">2296</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2297" href="#t2297">2297</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_gate</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers d_model d_mlp"</span><span class="op">]</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2298" href="#t2298">2298</a></span><span class="t">        <span class="str">"""Stack the MLP gate weights across all layers.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2299" href="#t2299">2299</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2300" href="#t2300">2300</a></span><span class="t"><span class="str">        Only works for models with gated MLPs.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2301" href="#t2301">2301</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2302" href="#t2302">2302</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">gated_mlp</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2303" href="#t2303">2303</a></span><span class="t">            <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">W_gate</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2304" href="#t2304">2304</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2305" href="#t2305">2305</a></span><span class="t">            <span class="key">return</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2306" href="#t2306">2306</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2307" href="#t2307">2307</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2308" href="#t2308">2308</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_out</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers d_mlp d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2309" href="#t2309">2309</a></span><span class="t">        <span class="str">"""Stack the MLP output weights across all layers."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2310" href="#t2310">2310</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">W_out</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2310&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2310&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2310 didn't run the list comprehension on line 2310, 2) line 2310 didn't return from function 'W_out', because the return on line 2310 wasn't executed</span></span></p>
+    <p class="pln"><span class="n"><a id="t2311" href="#t2311">2311</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2312" href="#t2312">2312</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2313" href="#t2313">2313</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_K</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2314" href="#t2314">2314</a></span><span class="t">        <span class="str">"""Stack the key biases across all layers."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2315" href="#t2315">2315</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">b_K</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2315&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2315&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2315 didn't run the list comprehension on line 2315, 2) line 2315 didn't return from function 'b_K', because the return on line 2315 wasn't executed</span></span></p>
+    <p class="pln"><span class="n"><a id="t2316" href="#t2316">2316</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2317" href="#t2317">2317</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2318" href="#t2318">2318</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_Q</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2319" href="#t2319">2319</a></span><span class="t">        <span class="str">"""Stack the query biases across all layers."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2320" href="#t2320">2320</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">b_Q</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2320&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2320&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2320 didn't run the list comprehension on line 2320, 2) line 2320 didn't return from function 'b_Q', because the return on line 2320 wasn't executed</span></span></p>
+    <p class="pln"><span class="n"><a id="t2321" href="#t2321">2321</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2322" href="#t2322">2322</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2323" href="#t2323">2323</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_V</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2324" href="#t2324">2324</a></span><span class="t">        <span class="str">"""Stack the value biases across all layers."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2325" href="#t2325">2325</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">b_V</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2325&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2325&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2325 didn't run the list comprehension on line 2325, 2) line 2325 didn't return from function 'b_V', because the return on line 2325 wasn't executed</span></span></p>
+    <p class="pln"><span class="n"><a id="t2326" href="#t2326">2326</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2327" href="#t2327">2327</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2328" href="#t2328">2328</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_O</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2329" href="#t2329">2329</a></span><span class="t">        <span class="str">"""Stack the attn output biases across all layers."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2330" href="#t2330">2330</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">b_O</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2330&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2330&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2330 didn't run the list comprehension on line 2330, 2) line 2330 didn't return from function 'b_O', because the return on line 2330 wasn't executed</span></span></p>
+    <p class="pln"><span class="n"><a id="t2331" href="#t2331">2331</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2332" href="#t2332">2332</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2333" href="#t2333">2333</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_in</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers d_mlp"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2334" href="#t2334">2334</a></span><span class="t">        <span class="str">"""Stack the MLP input biases across all layers."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2335" href="#t2335">2335</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">b_in</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2335&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2335&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2335 didn't run the list comprehension on line 2335, 2) line 2335 didn't return from function 'b_in', because the return on line 2335 wasn't executed</span></span></p>
+    <p class="pln"><span class="n"><a id="t2336" href="#t2336">2336</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2337" href="#t2337">2337</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2338" href="#t2338">2338</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_out</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2339" href="#t2339">2339</a></span><span class="t">        <span class="str">"""Stack the MLP output biases across all layers."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2340" href="#t2340">2340</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">b_out</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2340&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2340&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2340 didn't run the list comprehension on line 2340, 2) line 2340 didn't return from function 'b_out', because the return on line 2340 wasn't executed</span></span></p>
+    <p class="pln"><span class="n"><a id="t2341" href="#t2341">2341</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2342" href="#t2342">2342</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2343" href="#t2343">2343</a></span><span class="t">    <span class="key">def</span> <span class="nam">QK</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2344" href="#t2344">2344</a></span><span class="t">        <span class="key">return</span> <span class="nam">FactoredMatrix</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">W_Q</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">W_K</span><span class="op">.</span><span class="nam">transpose</span><span class="op">(</span><span class="op">-</span><span class="num">2</span><span class="op">,</span> <span class="op">-</span><span class="num">1</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2345" href="#t2345">2345</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2346" href="#t2346">2346</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2347" href="#t2347">2347</a></span><span class="t">    <span class="key">def</span> <span class="nam">OV</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2348" href="#t2348">2348</a></span><span class="t">        <span class="key">return</span> <span class="nam">FactoredMatrix</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">W_V</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">W_O</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2349" href="#t2349">2349</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2350" href="#t2350">2350</a></span><span class="t">    <span class="com"># Various utility functions</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2351" href="#t2351">2351</a></span><span class="t">    <span class="key">def</span> <span class="nam">accumulated_bias</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2352" href="#t2352">2352</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span> <span class="nam">layer</span><span class="op">:</span> <span class="nam">int</span><span class="op">,</span> <span class="nam">mlp_input</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span> <span class="nam">include_mlp_biases</span><span class="op">=</span><span class="key">True</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2353" href="#t2353">2353</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2354" href="#t2354">2354</a></span><span class="t">        <span class="str">"""Accumulated Bias.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t2355" href="#t2355">2355</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2356" href="#t2356">2356</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2357" href="#t2357">2357</a></span><span class="t"><span class="str">            layer (int): Layer number, in [0, n_layers]. layer==0 means no layers, layer==n_layers</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2358" href="#t2358">2358</a></span><span class="t"><span class="str">                means all layers.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2359" href="#t2359">2359</a></span><span class="t"><span class="str">            mlp_input (bool): If True, we take the bias up to the input of the MLP</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2360" href="#t2360">2360</a></span><span class="t"><span class="str">                of layer L (ie we include the bias from the attention output of the current layer,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2361" href="#t2361">2361</a></span><span class="t"><span class="str">                otherwise just biases from previous layers)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2362" href="#t2362">2362</a></span><span class="t"><span class="str">            include_mlp_biases (bool): Whether to include the biases of MLP layers. Often useful to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2363" href="#t2363">2363</a></span><span class="t"><span class="str">                have as False if we're expanding attn_out into individual heads, but keeping mlp_out</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2364" href="#t2364">2364</a></span><span class="t"><span class="str">                as is.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2365" href="#t2365">2365</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2366" href="#t2366">2366</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2367" href="#t2367">2367</a></span><span class="t"><span class="str">            bias (torch.Tensor): [d_model], accumulated bias</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2368" href="#t2368">2368</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2369" href="#t2369">2369</a></span><span class="t">        <span class="nam">accumulated_bias</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_model</span><span class="op">,</span> <span class="nam">device</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2370" href="#t2370">2370</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2371" href="#t2371">2371</a></span><span class="t">        <span class="key">for</span> <span class="nam">i</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">layer</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2372" href="#t2372">2372</a></span><span class="t">            <span class="nam">accumulated_bias</span> <span class="op">+=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">[</span><span class="nam">i</span><span class="op">]</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">b_O</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2373" href="#t2373">2373</a></span><span class="t">            <span class="key">if</span> <span class="nam">include_mlp_biases</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2374" href="#t2374">2374</a></span><span class="t">                <span class="nam">accumulated_bias</span> <span class="op">+=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">[</span><span class="nam">i</span><span class="op">]</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">b_out</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2375" href="#t2375">2375</a></span><span class="t">        <span class="key">if</span> <span class="nam">mlp_input</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2375&#x202F;&#x219B;&#x202F;2376</span><span class="annotate long">line 2375 didn't jump to line 2376, because the condition on line 2375 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2376" href="#t2376">2376</a></span><span class="t">            <span class="key">assert</span> <span class="nam">layer</span> <span class="op">&lt;</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">,</span> <span class="str">"Cannot include attn_bias from beyond the final layer"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2377" href="#t2377">2377</a></span><span class="t">            <span class="nam">accumulated_bias</span> <span class="op">+=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">[</span><span class="nam">layer</span><span class="op">]</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">b_O</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2378" href="#t2378">2378</a></span><span class="t">        <span class="key">return</span> <span class="nam">accumulated_bias</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2379" href="#t2379">2379</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2380" href="#t2380">2380</a></span><span class="t">    <span class="key">def</span> <span class="nam">all_composition_scores</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2381" href="#t2381">2381</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span> <span class="nam">mode</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2382" href="#t2382">2382</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads n_layers n_heads"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2383" href="#t2383">2383</a></span><span class="t">        <span class="str">"""All Composition Scores.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2384" href="#t2384">2384</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2385" href="#t2385">2385</a></span><span class="t"><span class="str">        Returns the Composition scores for all pairs of heads, as a L1, H1, L2, H2 tensor (which is</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2386" href="#t2386">2386</a></span><span class="t"><span class="str">        upper triangular on the first and third axes).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2356" href="#t2356">2356</a></span><span class="t"><span class="str">        Returns the accumulated bias from all layer outputs (ie the b_Os and b_outs), up to the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2357" href="#t2357">2357</a></span><span class="t"><span class="str">        input of layer L.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2358" href="#t2358">2358</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2359" href="#t2359">2359</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2360" href="#t2360">2360</a></span><span class="t"><span class="str">            layer (int): Layer number, in [0, n_layers]. layer==0 means no layers, layer==n_layers</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2361" href="#t2361">2361</a></span><span class="t"><span class="str">                means all layers.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2362" href="#t2362">2362</a></span><span class="t"><span class="str">            mlp_input (bool): If True, we take the bias up to the input of the MLP</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2363" href="#t2363">2363</a></span><span class="t"><span class="str">                of layer L (ie we include the bias from the attention output of the current layer,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2364" href="#t2364">2364</a></span><span class="t"><span class="str">                otherwise just biases from previous layers)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2365" href="#t2365">2365</a></span><span class="t"><span class="str">            include_mlp_biases (bool): Whether to include the biases of MLP layers. Often useful to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2366" href="#t2366">2366</a></span><span class="t"><span class="str">                have as False if we're expanding attn_out into individual heads, but keeping mlp_out</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2367" href="#t2367">2367</a></span><span class="t"><span class="str">                as is.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2368" href="#t2368">2368</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2369" href="#t2369">2369</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2370" href="#t2370">2370</a></span><span class="t"><span class="str">            bias (torch.Tensor): [d_model], accumulated bias</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2371" href="#t2371">2371</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2372" href="#t2372">2372</a></span><span class="t">        <span class="nam">accumulated_bias</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_model</span><span class="op">,</span> <span class="nam">device</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2373" href="#t2373">2373</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2374" href="#t2374">2374</a></span><span class="t">        <span class="key">for</span> <span class="nam">i</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">layer</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2375" href="#t2375">2375</a></span><span class="t">            <span class="nam">accumulated_bias</span> <span class="op">+=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">[</span><span class="nam">i</span><span class="op">]</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">b_O</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2376" href="#t2376">2376</a></span><span class="t">            <span class="key">if</span> <span class="nam">include_mlp_biases</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2377" href="#t2377">2377</a></span><span class="t">                <span class="nam">accumulated_bias</span> <span class="op">+=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">[</span><span class="nam">i</span><span class="op">]</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">b_out</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2378" href="#t2378">2378</a></span><span class="t">        <span class="key">if</span> <span class="nam">mlp_input</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2378&#x202F;&#x219B;&#x202F;2379</span><span class="annotate long">line 2378 didn't jump to line 2379, because the condition on line 2378 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2379" href="#t2379">2379</a></span><span class="t">            <span class="key">assert</span> <span class="nam">layer</span> <span class="op">&lt;</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">,</span> <span class="str">"Cannot include attn_bias from beyond the final layer"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2380" href="#t2380">2380</a></span><span class="t">            <span class="nam">accumulated_bias</span> <span class="op">+=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">[</span><span class="nam">layer</span><span class="op">]</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">b_O</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2381" href="#t2381">2381</a></span><span class="t">        <span class="key">return</span> <span class="nam">accumulated_bias</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2382" href="#t2382">2382</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2383" href="#t2383">2383</a></span><span class="t">    <span class="key">def</span> <span class="nam">all_composition_scores</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2384" href="#t2384">2384</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span> <span class="nam">mode</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2385" href="#t2385">2385</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads n_layers n_heads"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2386" href="#t2386">2386</a></span><span class="t">        <span class="str">"""All Composition Scores.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t2387" href="#t2387">2387</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2388" href="#t2388">2388</a></span><span class="t"><span class="str">        See</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2389" href="#t2389">2389</a></span><span class="t"><span class="str">        https://transformer-circuits.pub/2021/framework/index.html#:~:text=The%20above%20diagram%20shows%20Q%2D%2C%20K%2D%2C%20and%20V%2DComposition</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2390" href="#t2390">2390</a></span><span class="t"><span class="str">        for three metrics used.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2391" href="#t2391">2391</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2392" href="#t2392">2392</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2393" href="#t2393">2393</a></span><span class="t"><span class="str">            mode (str): One of ["Q", "K", "V"], the mode to use for the composition score.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2394" href="#t2394">2394</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2395" href="#t2395">2395</a></span><span class="t">        <span class="nam">left</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">OV</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2396" href="#t2396">2396</a></span><span class="t">        <span class="key">if</span> <span class="nam">mode</span> <span class="op">==</span> <span class="str">"Q"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2397" href="#t2397">2397</a></span><span class="t">            <span class="nam">right</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">QK</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2398" href="#t2398">2398</a></span><span class="t">        <span class="key">elif</span> <span class="nam">mode</span> <span class="op">==</span> <span class="str">"K"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2399" href="#t2399">2399</a></span><span class="t">            <span class="nam">right</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">QK</span><span class="op">.</span><span class="nam">T</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2400" href="#t2400">2400</a></span><span class="t">        <span class="key">elif</span> <span class="nam">mode</span> <span class="op">==</span> <span class="str">"V"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2401" href="#t2401">2401</a></span><span class="t">            <span class="nam">right</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">OV</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2402" href="#t2402">2402</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2403" href="#t2403">2403</a></span><span class="t">            <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">f"mode must be one of ['Q', 'K', 'V'] not {mode}"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2404" href="#t2404">2404</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2405" href="#t2405">2405</a></span><span class="t">        <span class="nam">scores</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">composition_scores</span><span class="op">(</span><span class="nam">left</span><span class="op">,</span> <span class="nam">right</span><span class="op">,</span> <span class="nam">broadcast_dims</span><span class="op">=</span><span class="key">True</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2406" href="#t2406">2406</a></span><span class="t">        <span class="com"># Mask scores to be zero for all pairs with the right head in the same layer or earlier</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2407" href="#t2407">2407</a></span><span class="t">        <span class="com"># layer than the left head.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2408" href="#t2408">2408</a></span><span class="t">        <span class="nam">mask</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2409" href="#t2409">2409</a></span><span class="t">            <span class="nam">torch</span><span class="op">.</span><span class="nam">arange</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">,</span> <span class="nam">device</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">,</span> <span class="key">None</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2410" href="#t2410">2410</a></span><span class="t">            <span class="op">&lt;</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">arange</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">,</span> <span class="nam">device</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span><span class="op">[</span><span class="key">None</span><span class="op">,</span> <span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2411" href="#t2411">2411</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2412" href="#t2412">2412</a></span><span class="t">        <span class="nam">scores</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">where</span><span class="op">(</span><span class="nam">mask</span><span class="op">,</span> <span class="nam">scores</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros_like</span><span class="op">(</span><span class="nam">scores</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2413" href="#t2413">2413</a></span><span class="t">        <span class="key">return</span> <span class="nam">scores</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2414" href="#t2414">2414</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2415" href="#t2415">2415</a></span><span class="t">    <span class="key">def</span> <span class="nam">all_head_labels</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2416" href="#t2416">2416</a></span><span class="t">        <span class="str">"""Returns a list of all head names in the model."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2417" href="#t2417">2417</a></span><span class="t">        <span class="key">return</span> <span class="op">[</span><span class="str">f"L{l}H{h}"</span> <span class="key">for</span> <span class="nam">l</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">)</span> <span class="key">for</span> <span class="nam">h</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_heads</span><span class="op">)</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2418" href="#t2418">2418</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2419" href="#t2419">2419</a></span><span class="t">    <span class="key">def</span> <span class="nam">load_sample_training_dataset</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2420" href="#t2420">2420</a></span><span class="t">        <span class="str">"""Load Sample Training Dataset.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2388" href="#t2388">2388</a></span><span class="t"><span class="str">        Returns the Composition scores for all pairs of heads, as a L1, H1, L2, H2 tensor (which is</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2389" href="#t2389">2389</a></span><span class="t"><span class="str">        upper triangular on the first and third axes).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2390" href="#t2390">2390</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2391" href="#t2391">2391</a></span><span class="t"><span class="str">        See</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2392" href="#t2392">2392</a></span><span class="t"><span class="str">        https://transformer-circuits.pub/2021/framework/index.html#:~:text=The%20above%20diagram%20shows%20Q%2D%2C%20K%2D%2C%20and%20V%2DComposition</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2393" href="#t2393">2393</a></span><span class="t"><span class="str">        for three metrics used.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2394" href="#t2394">2394</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2395" href="#t2395">2395</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2396" href="#t2396">2396</a></span><span class="t"><span class="str">            mode (str): One of ["Q", "K", "V"], the mode to use for the composition score.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2397" href="#t2397">2397</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2398" href="#t2398">2398</a></span><span class="t">        <span class="nam">left</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">OV</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2399" href="#t2399">2399</a></span><span class="t">        <span class="key">if</span> <span class="nam">mode</span> <span class="op">==</span> <span class="str">"Q"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2400" href="#t2400">2400</a></span><span class="t">            <span class="nam">right</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">QK</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2401" href="#t2401">2401</a></span><span class="t">        <span class="key">elif</span> <span class="nam">mode</span> <span class="op">==</span> <span class="str">"K"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2402" href="#t2402">2402</a></span><span class="t">            <span class="nam">right</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">QK</span><span class="op">.</span><span class="nam">T</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2403" href="#t2403">2403</a></span><span class="t">        <span class="key">elif</span> <span class="nam">mode</span> <span class="op">==</span> <span class="str">"V"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2404" href="#t2404">2404</a></span><span class="t">            <span class="nam">right</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">OV</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2405" href="#t2405">2405</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2406" href="#t2406">2406</a></span><span class="t">            <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">f"mode must be one of ['Q', 'K', 'V'] not {mode}"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2407" href="#t2407">2407</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2408" href="#t2408">2408</a></span><span class="t">        <span class="nam">scores</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">composition_scores</span><span class="op">(</span><span class="nam">left</span><span class="op">,</span> <span class="nam">right</span><span class="op">,</span> <span class="nam">broadcast_dims</span><span class="op">=</span><span class="key">True</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2409" href="#t2409">2409</a></span><span class="t">        <span class="com"># Mask scores to be zero for all pairs with the right head in the same layer or earlier</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2410" href="#t2410">2410</a></span><span class="t">        <span class="com"># layer than the left head.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2411" href="#t2411">2411</a></span><span class="t">        <span class="nam">mask</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2412" href="#t2412">2412</a></span><span class="t">            <span class="nam">torch</span><span class="op">.</span><span class="nam">arange</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">,</span> <span class="nam">device</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">,</span> <span class="key">None</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2413" href="#t2413">2413</a></span><span class="t">            <span class="op">&lt;</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">arange</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">,</span> <span class="nam">device</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span><span class="op">[</span><span class="key">None</span><span class="op">,</span> <span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2414" href="#t2414">2414</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2415" href="#t2415">2415</a></span><span class="t">        <span class="nam">scores</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">where</span><span class="op">(</span><span class="nam">mask</span><span class="op">,</span> <span class="nam">scores</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros_like</span><span class="op">(</span><span class="nam">scores</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2416" href="#t2416">2416</a></span><span class="t">        <span class="key">return</span> <span class="nam">scores</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2417" href="#t2417">2417</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2418" href="#t2418">2418</a></span><span class="t">    <span class="key">def</span> <span class="nam">all_head_labels</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2419" href="#t2419">2419</a></span><span class="t">        <span class="str">"""Returns a list of all head names in the model."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2420" href="#t2420">2420</a></span><span class="t">        <span class="key">return</span> <span class="op">[</span><span class="str">f"L{l}H{h}"</span> <span class="key">for</span> <span class="nam">l</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">)</span> <span class="key">for</span> <span class="nam">h</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_heads</span><span class="op">)</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t2421" href="#t2421">2421</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2422" href="#t2422">2422</a></span><span class="t"><span class="str">        Helper function to load in a 10K-20K dataset of elements from the model's training data</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2423" href="#t2423">2423</a></span><span class="t"><span class="str">        distribution.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2422" href="#t2422">2422</a></span><span class="t">    <span class="key">def</span> <span class="nam">load_sample_training_dataset</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2423" href="#t2423">2423</a></span><span class="t">        <span class="str">"""Load Sample Training Dataset.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t2424" href="#t2424">2424</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2425" href="#t2425">2425</a></span><span class="t"><span class="str">        Wrapper around utils.get_dataset, which identifies the appropriate dataset the pretrained</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2426" href="#t2426">2426</a></span><span class="t"><span class="str">        models. Each dataset has a 'text' field, which contains the relevant info, some have several</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2427" href="#t2427">2427</a></span><span class="t"><span class="str">        meta data fields.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2428" href="#t2428">2428</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2429" href="#t2429">2429</a></span><span class="t"><span class="str">        Kwargs will be passed to utils.get_dataset (e.g. cache_dir to set download location)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2430" href="#t2430">2430</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2431" href="#t2431">2431</a></span><span class="t"><span class="str">        Notes:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2432" href="#t2432">2432</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2433" href="#t2433">2433</a></span><span class="t"><span class="str">        - PT-2's training data is not open source. OpenWebText is a replication (links with</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2434" href="#t2434">2434</a></span><span class="t"><span class="str">            >3 karma on Reddit)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2435" href="#t2435">2435</a></span><span class="t"><span class="str">        - OPT's training data is not open source, and is a mess of different things that is hard to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2436" href="#t2436">2436</a></span><span class="t"><span class="str">          replicate. I default to the Pile, which covers some of it, but imperfectly.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2437" href="#t2437">2437</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2438" href="#t2438">2438</a></span><span class="t"><span class="str">        (Some models will have actually been trained on the data supplied here, for some it's from</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2439" href="#t2439">2439</a></span><span class="t"><span class="str">        the validation set).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2440" href="#t2440">2440</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2441" href="#t2441">2441</a></span><span class="t">        <span class="nam">model_dataset_map</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2442" href="#t2442">2442</a></span><span class="t">            <span class="str">"neel"</span><span class="op">:</span> <span class="str">"c4_code"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2443" href="#t2443">2443</a></span><span class="t">            <span class="str">"neel-solu-old"</span><span class="op">:</span> <span class="str">"pile"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2444" href="#t2444">2444</a></span><span class="t">            <span class="str">"GPT2LMHeadModel"</span><span class="op">:</span> <span class="str">"openwebtext"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2445" href="#t2445">2445</a></span><span class="t">            <span class="str">"GPTNeoForCausalLM"</span><span class="op">:</span> <span class="str">"pile"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2446" href="#t2446">2446</a></span><span class="t">            <span class="str">"GPTNeoXForCausalLM"</span><span class="op">:</span> <span class="str">"pile"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2447" href="#t2447">2447</a></span><span class="t">            <span class="str">"GPTJForCausalLM"</span><span class="op">:</span> <span class="str">"pile"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2448" href="#t2448">2448</a></span><span class="t">            <span class="str">"OPTForCausalLM"</span><span class="op">:</span> <span class="str">"pile"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2449" href="#t2449">2449</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2450" href="#t2450">2450</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="key">in</span> <span class="nam">model_dataset_map</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2451" href="#t2451">2451</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">get_dataset</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2452" href="#t2452">2452</a></span><span class="t">                <span class="nam">model_dataset_map</span><span class="op">[</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span><span class="op">]</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2453" href="#t2453">2453</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2454" href="#t2454">2454</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2455" href="#t2455">2455</a></span><span class="t">            <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2456" href="#t2456">2456</a></span><span class="t">                <span class="str">f"We do not have an available dataset for the relevant model: {self.cfg.original_architecture}"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2457" href="#t2457">2457</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2458" href="#t2458">2458</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2459" href="#t2459">2459</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2460" href="#t2460">2460</a></span><span class="t">    <span class="key">def</span> <span class="nam">sample_datapoint</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2461" href="#t2461">2461</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2462" href="#t2462">2462</a></span><span class="t">        <span class="nam">tokenize</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2463" href="#t2463">2463</a></span><span class="t">        <span class="nam">prepend_bos</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">bool</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2464" href="#t2464">2464</a></span><span class="t">        <span class="nam">padding_side</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Literal</span><span class="op">[</span><span class="str">"left"</span><span class="op">,</span> <span class="str">"right"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2465" href="#t2465">2465</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Union</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"1 pos"</span><span class="op">]</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2466" href="#t2466">2466</a></span><span class="t">        <span class="str">"""Sample Data Point from Dataset.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2467" href="#t2467">2467</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2468" href="#t2468">2468</a></span><span class="t"><span class="str">        Helper function to randomly sample a data point from self.dataset, a small dataset from the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2469" href="#t2469">2469</a></span><span class="t"><span class="str">        data distribution the model was trained on.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2425" href="#t2425">2425</a></span><span class="t"><span class="str">        Helper function to load in a 10K-20K dataset of elements from the model's training data</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2426" href="#t2426">2426</a></span><span class="t"><span class="str">        distribution.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2427" href="#t2427">2427</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2428" href="#t2428">2428</a></span><span class="t"><span class="str">        Wrapper around utils.get_dataset, which identifies the appropriate dataset the pretrained</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2429" href="#t2429">2429</a></span><span class="t"><span class="str">        models. Each dataset has a 'text' field, which contains the relevant info, some have several</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2430" href="#t2430">2430</a></span><span class="t"><span class="str">        meta data fields.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2431" href="#t2431">2431</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2432" href="#t2432">2432</a></span><span class="t"><span class="str">        Kwargs will be passed to utils.get_dataset (e.g. cache_dir to set download location)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2433" href="#t2433">2433</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2434" href="#t2434">2434</a></span><span class="t"><span class="str">        Notes:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2435" href="#t2435">2435</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2436" href="#t2436">2436</a></span><span class="t"><span class="str">        - PT-2's training data is not open source. OpenWebText is a replication (links with</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2437" href="#t2437">2437</a></span><span class="t"><span class="str">            >3 karma on Reddit)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2438" href="#t2438">2438</a></span><span class="t"><span class="str">        - OPT's training data is not open source, and is a mess of different things that is hard to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2439" href="#t2439">2439</a></span><span class="t"><span class="str">          replicate. I default to the Pile, which covers some of it, but imperfectly.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2440" href="#t2440">2440</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2441" href="#t2441">2441</a></span><span class="t"><span class="str">        (Some models will have actually been trained on the data supplied here, for some it's from</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2442" href="#t2442">2442</a></span><span class="t"><span class="str">        the validation set).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2443" href="#t2443">2443</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2444" href="#t2444">2444</a></span><span class="t">        <span class="nam">model_dataset_map</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2445" href="#t2445">2445</a></span><span class="t">            <span class="str">"neel"</span><span class="op">:</span> <span class="str">"c4_code"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2446" href="#t2446">2446</a></span><span class="t">            <span class="str">"neel-solu-old"</span><span class="op">:</span> <span class="str">"pile"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2447" href="#t2447">2447</a></span><span class="t">            <span class="str">"GPT2LMHeadModel"</span><span class="op">:</span> <span class="str">"openwebtext"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2448" href="#t2448">2448</a></span><span class="t">            <span class="str">"GPTNeoForCausalLM"</span><span class="op">:</span> <span class="str">"pile"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2449" href="#t2449">2449</a></span><span class="t">            <span class="str">"GPTNeoXForCausalLM"</span><span class="op">:</span> <span class="str">"pile"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2450" href="#t2450">2450</a></span><span class="t">            <span class="str">"GPTJForCausalLM"</span><span class="op">:</span> <span class="str">"pile"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2451" href="#t2451">2451</a></span><span class="t">            <span class="str">"OPTForCausalLM"</span><span class="op">:</span> <span class="str">"pile"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2452" href="#t2452">2452</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2453" href="#t2453">2453</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="key">in</span> <span class="nam">model_dataset_map</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2454" href="#t2454">2454</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">get_dataset</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2455" href="#t2455">2455</a></span><span class="t">                <span class="nam">model_dataset_map</span><span class="op">[</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span><span class="op">]</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2456" href="#t2456">2456</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2457" href="#t2457">2457</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2458" href="#t2458">2458</a></span><span class="t">            <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2459" href="#t2459">2459</a></span><span class="t">                <span class="str">f"We do not have an available dataset for the relevant model: {self.cfg.original_architecture}"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2460" href="#t2460">2460</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2461" href="#t2461">2461</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2462" href="#t2462">2462</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2463" href="#t2463">2463</a></span><span class="t">    <span class="key">def</span> <span class="nam">sample_datapoint</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2464" href="#t2464">2464</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2465" href="#t2465">2465</a></span><span class="t">        <span class="nam">tokenize</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2466" href="#t2466">2466</a></span><span class="t">        <span class="nam">prepend_bos</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">bool</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2467" href="#t2467">2467</a></span><span class="t">        <span class="nam">padding_side</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Literal</span><span class="op">[</span><span class="str">"left"</span><span class="op">,</span> <span class="str">"right"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2468" href="#t2468">2468</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Union</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"1 pos"</span><span class="op">]</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2469" href="#t2469">2469</a></span><span class="t">        <span class="str">"""Sample Data Point from Dataset.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t2470" href="#t2470">2470</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2471" href="#t2471">2471</a></span><span class="t"><span class="str">        Implicitly calls self.load_sample_training_dataset if it hasn't already been called. Only</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2472" href="#t2472">2472</a></span><span class="t"><span class="str">        works for pretrained models with an associated dataset. But you can manually replace</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2473" href="#t2473">2473</a></span><span class="t"><span class="str">        self.dataset with a dataset of your choice if you want.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2474" href="#t2474">2474</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2475" href="#t2475">2475</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2476" href="#t2476">2476</a></span><span class="t"><span class="str">            tokenize (bool): Whether to return tokens (instead of text). Defaults to False. Note</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2477" href="#t2477">2477</a></span><span class="t"><span class="str">                that the returned tokens will be automatically truncated to the model's max context</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2478" href="#t2478">2478</a></span><span class="t"><span class="str">                size.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2479" href="#t2479">2479</a></span><span class="t"><span class="str">            prepend_bos (bool, optional): Overrides self.cfg.default_prepend_bos. Whether to prepend</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2480" href="#t2480">2480</a></span><span class="t"><span class="str">                the BOS token to the input (applicable when input is a string). Defaults to None,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2481" href="#t2481">2481</a></span><span class="t"><span class="str">                implying usage of self.cfg.default_prepend_bos (default is True unless specified</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2482" href="#t2482">2482</a></span><span class="t"><span class="str">                otherwise). Pass True or False to override the default.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2483" href="#t2483">2483</a></span><span class="t"><span class="str">            padding_side (Union[Literal["left", "right"], None], optional): Overrides</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2484" href="#t2484">2484</a></span><span class="t"><span class="str">                self.tokenizer.padding_side. Specifies which side to pad when tokenizing multiple</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2485" href="#t2485">2485</a></span><span class="t"><span class="str">                strings of different lengths.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2486" href="#t2486">2486</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2487" href="#t2487">2487</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2488" href="#t2488">2488</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">load_sample_training_dataset</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2489" href="#t2489">2489</a></span><span class="t">        <span class="key">assert</span> <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>  <span class="com"># keep mypy happy</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2490" href="#t2490">2490</a></span><span class="t">        <span class="nam">sample_dataset_size</span> <span class="op">=</span> <span class="nam">len</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2491" href="#t2491">2491</a></span><span class="t">        <span class="nam">index</span> <span class="op">=</span> <span class="nam">np</span><span class="op">.</span><span class="nam">random</span><span class="op">.</span><span class="nam">randint</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">sample_dataset_size</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2492" href="#t2492">2492</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">tokenize</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2493" href="#t2493">2493</a></span><span class="t">            <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span><span class="op">[</span><span class="nam">index</span><span class="op">]</span><span class="op">[</span><span class="str">"text"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2494" href="#t2494">2494</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2495" href="#t2495">2495</a></span><span class="t">            <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">to_tokens</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2496" href="#t2496">2496</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span><span class="op">[</span><span class="nam">index</span><span class="op">]</span><span class="op">[</span><span class="str">"text"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2497" href="#t2497">2497</a></span><span class="t">                <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2498" href="#t2498">2498</a></span><span class="t">                <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2499" href="#t2499">2499</a></span><span class="t">                <span class="nam">truncate</span><span class="op">=</span><span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2500" href="#t2500">2500</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2471" href="#t2471">2471</a></span><span class="t"><span class="str">        Helper function to randomly sample a data point from self.dataset, a small dataset from the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2472" href="#t2472">2472</a></span><span class="t"><span class="str">        data distribution the model was trained on.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2473" href="#t2473">2473</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2474" href="#t2474">2474</a></span><span class="t"><span class="str">        Implicitly calls self.load_sample_training_dataset if it hasn't already been called. Only</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2475" href="#t2475">2475</a></span><span class="t"><span class="str">        works for pretrained models with an associated dataset. But you can manually replace</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2476" href="#t2476">2476</a></span><span class="t"><span class="str">        self.dataset with a dataset of your choice if you want.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2477" href="#t2477">2477</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2478" href="#t2478">2478</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2479" href="#t2479">2479</a></span><span class="t"><span class="str">            tokenize (bool): Whether to return tokens (instead of text). Defaults to False. Note</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2480" href="#t2480">2480</a></span><span class="t"><span class="str">                that the returned tokens will be automatically truncated to the model's max context</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2481" href="#t2481">2481</a></span><span class="t"><span class="str">                size.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2482" href="#t2482">2482</a></span><span class="t"><span class="str">            prepend_bos (bool, optional): Overrides self.cfg.default_prepend_bos. Whether to prepend</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2483" href="#t2483">2483</a></span><span class="t"><span class="str">                the BOS token to the input (applicable when input is a string). Defaults to None,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2484" href="#t2484">2484</a></span><span class="t"><span class="str">                implying usage of self.cfg.default_prepend_bos (default is True unless specified</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2485" href="#t2485">2485</a></span><span class="t"><span class="str">                otherwise). Pass True or False to override the default.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2486" href="#t2486">2486</a></span><span class="t"><span class="str">            padding_side (Union[Literal["left", "right"], None], optional): Overrides</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2487" href="#t2487">2487</a></span><span class="t"><span class="str">                self.tokenizer.padding_side. Specifies which side to pad when tokenizing multiple</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2488" href="#t2488">2488</a></span><span class="t"><span class="str">                strings of different lengths.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2489" href="#t2489">2489</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2490" href="#t2490">2490</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2491" href="#t2491">2491</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">load_sample_training_dataset</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2492" href="#t2492">2492</a></span><span class="t">        <span class="key">assert</span> <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>  <span class="com"># keep mypy happy</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2493" href="#t2493">2493</a></span><span class="t">        <span class="nam">sample_dataset_size</span> <span class="op">=</span> <span class="nam">len</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2494" href="#t2494">2494</a></span><span class="t">        <span class="nam">index</span> <span class="op">=</span> <span class="nam">np</span><span class="op">.</span><span class="nam">random</span><span class="op">.</span><span class="nam">randint</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">sample_dataset_size</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2495" href="#t2495">2495</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">tokenize</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2496" href="#t2496">2496</a></span><span class="t">            <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span><span class="op">[</span><span class="nam">index</span><span class="op">]</span><span class="op">[</span><span class="str">"text"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2497" href="#t2497">2497</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2498" href="#t2498">2498</a></span><span class="t">            <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">to_tokens</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2499" href="#t2499">2499</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span><span class="op">[</span><span class="nam">index</span><span class="op">]</span><span class="op">[</span><span class="str">"text"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2500" href="#t2500">2500</a></span><span class="t">                <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2501" href="#t2501">2501</a></span><span class="t">                <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2502" href="#t2502">2502</a></span><span class="t">                <span class="nam">truncate</span><span class="op">=</span><span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2503" href="#t2503">2503</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
 </main>
 <footer>
     <div class="content">
@@ -2591,7 +2594,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_HookedTransformerConfig_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_SVDInterpreter_py.html b/_static/coverage/d_af97b5493da09a14_SVDInterpreter_py.html
index e6875a453..20af581ef 100644
--- a/_static/coverage/d_af97b5493da09a14_SVDInterpreter_py.html
+++ b/_static/coverage/d_af97b5493da09a14_SVDInterpreter_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14___init___py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -252,7 +252,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14___init___py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14___init___py.html b/_static/coverage/d_af97b5493da09a14___init___py.html
index fcd15ded1..fc600f14e 100644
--- a/_static/coverage/d_af97b5493da09a14___init___py.html
+++ b/_static/coverage/d_af97b5493da09a14___init___py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831___init___py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -118,7 +118,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831___init___py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_evals_py.html b/_static/coverage/d_af97b5493da09a14_evals_py.html
index 65d0dfaac..79ac78ee3 100644
--- a/_static/coverage/d_af97b5493da09a14_evals_py.html
+++ b/_static/coverage/d_af97b5493da09a14_evals_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_65d4430f90bfb219_activation_function_factory_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -427,7 +427,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_65d4430f90bfb219_activation_function_factory_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_head_detector_py.html b/_static/coverage/d_af97b5493da09a14_head_detector_py.html
index 395e1148a..835188e5a 100644
--- a/_static/coverage/d_af97b5493da09a14_head_detector_py.html
+++ b/_static/coverage/d_af97b5493da09a14_head_detector_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_hook_points_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -361,7 +361,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_hook_points_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_hook_points_py.html b/_static/coverage/d_af97b5493da09a14_hook_points_py.html
index bed02cd06..ea15c0ac5 100644
--- a/_static/coverage/d_af97b5493da09a14_hook_points_py.html
+++ b/_static/coverage/d_af97b5493da09a14_hook_points_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_loading_from_pretrained_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -789,7 +789,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_loading_from_pretrained_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_loading_from_pretrained_py.html b/_static/coverage/d_af97b5493da09a14_loading_from_pretrained_py.html
index 371ec0a48..f2a275b55 100644
--- a/_static/coverage/d_af97b5493da09a14_loading_from_pretrained_py.html
+++ b/_static/coverage/d_af97b5493da09a14_loading_from_pretrained_py.html
@@ -2,7 +2,7 @@
 <html>
 <head>
     <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
-    <title>Coverage for transformer_lens/loading_from_pretrained.py: 62%</title>
+    <title>Coverage for transformer_lens/loading_from_pretrained.py: 64%</title>
     <link rel="icon" sizes="32x32" href="favicon_32.png">
     <link rel="stylesheet" href="style.css" type="text/css">
     <script type="text/javascript" src="coverage_html.js" defer></script>
@@ -12,7 +12,7 @@
     <div class="content">
         <h1>
             <span class="text">Coverage for </span><b>transformer_lens/loading_from_pretrained.py</b>:
-            <span class="pc_cov">62%</span>
+            <span class="pc_cov">64%</span>
         </h1>
         <aside id="help_panel_wrapper">
             <input id="help_panel_state" type="checkbox">
@@ -55,11 +55,11 @@ <h1>
             </div>
         </aside>
         <h2>
-            <span class="text">320 statements &nbsp;</span>
-            <button type="button" class="run button_toggle_run" value="run" data-shortcut="r" title="Toggle lines run">208<span class="text"> run</span></button>
-            <button type="button" class="mis show_mis button_toggle_mis" value="mis" data-shortcut="m" title="Toggle lines missing">112<span class="text"> missing</span></button>
+            <span class="text">322 statements &nbsp;</span>
+            <button type="button" class="run button_toggle_run" value="run" data-shortcut="r" title="Toggle lines run">216<span class="text"> run</span></button>
+            <button type="button" class="mis show_mis button_toggle_mis" value="mis" data-shortcut="m" title="Toggle lines missing">106<span class="text"> missing</span></button>
             <button type="button" class="exc show_exc button_toggle_exc" value="exc" data-shortcut="x" title="Toggle lines excluded">0<span class="text"> excluded</span></button>
-            <button type="button" class="par run show_par button_toggle_par" value="par" data-shortcut="p" title="Toggle lines partially run">51<span class="text"> partial</span></button>
+            <button type="button" class="par run show_par button_toggle_par" value="par" data-shortcut="p" title="Toggle lines partially run">53<span class="text"> partial</span></button>
         </h2>
         <p class="text">
             <a id="prevFileLink" class="nav" href="d_af97b5493da09a14_hook_points_py.html">&#xab; prev</a> &nbsp; &nbsp;
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_past_key_value_caching_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -235,1245 +235,1245 @@ <h2>
     <p class="pln"><span class="n"><a id="t151" href="#t151">151</a></span><span class="t">    <span class="str">"meta-llama/Meta-Llama-3-8B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t152" href="#t152">152</a></span><span class="t">    <span class="str">"meta-llama/Meta-Llama-3-70B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t153" href="#t153">153</a></span><span class="t">    <span class="str">"meta-llama/Meta-Llama-3-70B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t154" href="#t154">154</a></span><span class="t">    <span class="str">"meta-llama/Llama-3.2-1B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t155" href="#t155">155</a></span><span class="t">    <span class="str">"meta-llama/Llama-3.2-3B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t156" href="#t156">156</a></span><span class="t">    <span class="str">"meta-llama/Llama-3.2-1B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t157" href="#t157">157</a></span><span class="t">    <span class="str">"meta-llama/Llama-3.2-3B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t158" href="#t158">158</a></span><span class="t">    <span class="str">"meta-llama/Llama-3.1-70B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t159" href="#t159">159</a></span><span class="t">    <span class="str">"meta-llama/Llama-3.1-8B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t160" href="#t160">160</a></span><span class="t">    <span class="str">"meta-llama/Llama-3.1-8B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t161" href="#t161">161</a></span><span class="t">    <span class="str">"meta-llama/Llama-3.1-70B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t162" href="#t162">162</a></span><span class="t">    <span class="str">"Baidicoot/Othello-GPT-Transformer-Lens"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t163" href="#t163">163</a></span><span class="t">    <span class="str">"bert-base-cased"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t164" href="#t164">164</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-1M"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t165" href="#t165">165</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-3M"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t166" href="#t166">166</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-8M"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t167" href="#t167">167</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-28M"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t168" href="#t168">168</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-33M"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t169" href="#t169">169</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instruct-1M"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t170" href="#t170">170</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instruct-3M"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t171" href="#t171">171</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instruct-8M"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t172" href="#t172">172</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instruct-28M"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t173" href="#t173">173</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instruct-33M"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t174" href="#t174">174</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-1Layer-21M"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t175" href="#t175">175</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-2Layers-33M"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t176" href="#t176">176</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instuct-1Layer-21M"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t177" href="#t177">177</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instruct-2Layers-33M"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t178" href="#t178">178</a></span><span class="t">    <span class="str">"stabilityai/stablelm-base-alpha-3b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t179" href="#t179">179</a></span><span class="t">    <span class="str">"stabilityai/stablelm-base-alpha-7b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t180" href="#t180">180</a></span><span class="t">    <span class="str">"stabilityai/stablelm-tuned-alpha-3b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t181" href="#t181">181</a></span><span class="t">    <span class="str">"stabilityai/stablelm-tuned-alpha-7b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t182" href="#t182">182</a></span><span class="t">    <span class="str">"mistralai/Mistral-7B-v0.1"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t183" href="#t183">183</a></span><span class="t">    <span class="str">"mistralai/Mistral-7B-Instruct-v0.1"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t184" href="#t184">184</a></span><span class="t">    <span class="str">"mistralai/Mistral-Nemo-Base-2407"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t185" href="#t185">185</a></span><span class="t">    <span class="str">"mistralai/Mixtral-8x7B-v0.1"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t186" href="#t186">186</a></span><span class="t">    <span class="str">"mistralai/Mixtral-8x7B-Instruct-v0.1"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t187" href="#t187">187</a></span><span class="t">    <span class="str">"bigscience/bloom-560m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t188" href="#t188">188</a></span><span class="t">    <span class="str">"bigscience/bloom-1b1"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t189" href="#t189">189</a></span><span class="t">    <span class="str">"bigscience/bloom-1b7"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t190" href="#t190">190</a></span><span class="t">    <span class="str">"bigscience/bloom-3b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t191" href="#t191">191</a></span><span class="t">    <span class="str">"bigscience/bloom-7b1"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t192" href="#t192">192</a></span><span class="t">    <span class="str">"bigcode/santacoder"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t193" href="#t193">193</a></span><span class="t">    <span class="str">"Qwen/Qwen-1_8B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t194" href="#t194">194</a></span><span class="t">    <span class="str">"Qwen/Qwen-7B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t195" href="#t195">195</a></span><span class="t">    <span class="str">"Qwen/Qwen-14B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t196" href="#t196">196</a></span><span class="t">    <span class="str">"Qwen/Qwen-1_8B-Chat"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t197" href="#t197">197</a></span><span class="t">    <span class="str">"Qwen/Qwen-7B-Chat"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t198" href="#t198">198</a></span><span class="t">    <span class="str">"Qwen/Qwen-14B-Chat"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t199" href="#t199">199</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-0.5B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t200" href="#t200">200</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-0.5B-Chat"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t201" href="#t201">201</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-1.8B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t202" href="#t202">202</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-1.8B-Chat"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t203" href="#t203">203</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-4B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t204" href="#t204">204</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-4B-Chat"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t205" href="#t205">205</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-7B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t206" href="#t206">206</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-7B-Chat"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t207" href="#t207">207</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-14B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t208" href="#t208">208</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-14B-Chat"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t209" href="#t209">209</a></span><span class="t">    <span class="str">"Qwen/Qwen2-0.5B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t210" href="#t210">210</a></span><span class="t">    <span class="str">"Qwen/Qwen2-0.5B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t211" href="#t211">211</a></span><span class="t">    <span class="str">"Qwen/Qwen2-1.5B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t212" href="#t212">212</a></span><span class="t">    <span class="str">"Qwen/Qwen2-1.5B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t213" href="#t213">213</a></span><span class="t">    <span class="str">"Qwen/Qwen2-7B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t214" href="#t214">214</a></span><span class="t">    <span class="str">"Qwen/Qwen2-7B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t215" href="#t215">215</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-0.5B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t216" href="#t216">216</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-0.5B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t217" href="#t217">217</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-1.5B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t218" href="#t218">218</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-1.5B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t219" href="#t219">219</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-3B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t220" href="#t220">220</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-3B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t221" href="#t221">221</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-7B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t222" href="#t222">222</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-7B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t223" href="#t223">223</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-14B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t224" href="#t224">224</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-14B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t225" href="#t225">225</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-32B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t226" href="#t226">226</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-32B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t227" href="#t227">227</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-72B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t228" href="#t228">228</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-72B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t229" href="#t229">229</a></span><span class="t">    <span class="str">"Qwen/QwQ-32B-Preview"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t230" href="#t230">230</a></span><span class="t">    <span class="str">"microsoft/phi-1"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t231" href="#t231">231</a></span><span class="t">    <span class="str">"microsoft/phi-1_5"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t232" href="#t232">232</a></span><span class="t">    <span class="str">"microsoft/phi-2"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t233" href="#t233">233</a></span><span class="t">    <span class="str">"microsoft/Phi-3-mini-4k-instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t234" href="#t234">234</a></span><span class="t">    <span class="str">"google/gemma-2b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t235" href="#t235">235</a></span><span class="t">    <span class="str">"google/gemma-7b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t236" href="#t236">236</a></span><span class="t">    <span class="str">"google/gemma-2b-it"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t237" href="#t237">237</a></span><span class="t">    <span class="str">"google/gemma-7b-it"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t238" href="#t238">238</a></span><span class="t">    <span class="str">"google/gemma-2-2b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t239" href="#t239">239</a></span><span class="t">    <span class="str">"google/gemma-2-2b-it"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t240" href="#t240">240</a></span><span class="t">    <span class="str">"google/gemma-2-9b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t241" href="#t241">241</a></span><span class="t">    <span class="str">"google/gemma-2-9b-it"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t242" href="#t242">242</a></span><span class="t">    <span class="str">"google/gemma-2-27b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t243" href="#t243">243</a></span><span class="t">    <span class="str">"google/gemma-2-27b-it"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t244" href="#t244">244</a></span><span class="t">    <span class="str">"01-ai/Yi-6B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t245" href="#t245">245</a></span><span class="t">    <span class="str">"01-ai/Yi-34B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t246" href="#t246">246</a></span><span class="t">    <span class="str">"01-ai/Yi-6B-Chat"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t247" href="#t247">247</a></span><span class="t">    <span class="str">"01-ai/Yi-34B-Chat"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t248" href="#t248">248</a></span><span class="t">    <span class="str">"google-t5/t5-small"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t249" href="#t249">249</a></span><span class="t">    <span class="str">"google-t5/t5-base"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t250" href="#t250">250</a></span><span class="t">    <span class="str">"google-t5/t5-large"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t251" href="#t251">251</a></span><span class="t">    <span class="str">"ai-forever/mGPT"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t252" href="#t252">252</a></span><span class="t"><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t253" href="#t253">253</a></span><span class="t"><span class="str">"""Official model names for models on HuggingFace."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t254" href="#t254">254</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t255" href="#t255">255</a></span><span class="t"><span class="com"># Model Aliases:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t256" href="#t256">256</a></span><span class="t"><span class="nam">MODEL_ALIASES</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t257" href="#t257">257</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_1L_v9_old"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-1l-pile"</span><span class="op">,</span> <span class="str">"solu-1l-old"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t258" href="#t258">258</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_2L_v10_old"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-2l-pile"</span><span class="op">,</span> <span class="str">"solu-2l-old"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t259" href="#t259">259</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_4L_v11_old"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-4l-pile"</span><span class="op">,</span> <span class="str">"solu-4l-old"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t260" href="#t260">260</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_6L_v13_old"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-6l-pile"</span><span class="op">,</span> <span class="str">"solu-6l-old"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t261" href="#t261">261</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_8L_v21_old"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-8l-pile"</span><span class="op">,</span> <span class="str">"solu-8l-old"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t262" href="#t262">262</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_10L_v22_old"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-10l-pile"</span><span class="op">,</span> <span class="str">"solu-10l-old"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t263" href="#t263">263</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_12L_v23_old"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-12l-pile"</span><span class="op">,</span> <span class="str">"solu-12l-old"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t264" href="#t264">264</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_1L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-1l"</span><span class="op">,</span> <span class="str">"solu-1l-new"</span><span class="op">,</span> <span class="str">"solu-1l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t265" href="#t265">265</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_2L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-2l"</span><span class="op">,</span> <span class="str">"solu-2l-new"</span><span class="op">,</span> <span class="str">"solu-2l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t266" href="#t266">266</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_3L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-3l"</span><span class="op">,</span> <span class="str">"solu-3l-new"</span><span class="op">,</span> <span class="str">"solu-3l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t267" href="#t267">267</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_4L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-4l"</span><span class="op">,</span> <span class="str">"solu-4l-new"</span><span class="op">,</span> <span class="str">"solu-4l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t268" href="#t268">268</a></span><span class="t">    <span class="str">"NeelNanda/GELU_1L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gelu-1l"</span><span class="op">,</span> <span class="str">"gelu-1l-new"</span><span class="op">,</span> <span class="str">"gelu-1l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t269" href="#t269">269</a></span><span class="t">    <span class="str">"NeelNanda/GELU_2L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gelu-2l"</span><span class="op">,</span> <span class="str">"gelu-2l-new"</span><span class="op">,</span> <span class="str">"gelu-2l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t270" href="#t270">270</a></span><span class="t">    <span class="str">"NeelNanda/GELU_3L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gelu-3l"</span><span class="op">,</span> <span class="str">"gelu-3l-new"</span><span class="op">,</span> <span class="str">"gelu-3l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t271" href="#t271">271</a></span><span class="t">    <span class="str">"NeelNanda/GELU_4L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gelu-4l"</span><span class="op">,</span> <span class="str">"gelu-4l-new"</span><span class="op">,</span> <span class="str">"gelu-4l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t272" href="#t272">272</a></span><span class="t">    <span class="str">"NeelNanda/Attn_Only_1L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t273" href="#t273">273</a></span><span class="t">        <span class="str">"attn-only-1l"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t274" href="#t274">274</a></span><span class="t">        <span class="str">"attn-only-1l-new"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t275" href="#t275">275</a></span><span class="t">        <span class="str">"attn-only-1l-c4-code"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t276" href="#t276">276</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t277" href="#t277">277</a></span><span class="t">    <span class="str">"NeelNanda/Attn_Only_2L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t278" href="#t278">278</a></span><span class="t">        <span class="str">"attn-only-2l"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t279" href="#t279">279</a></span><span class="t">        <span class="str">"attn-only-2l-new"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t280" href="#t280">280</a></span><span class="t">        <span class="str">"attn-only-2l-c4-code"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t281" href="#t281">281</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t282" href="#t282">282</a></span><span class="t">    <span class="str">"NeelNanda/Attn_Only_3L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t283" href="#t283">283</a></span><span class="t">        <span class="str">"attn-only-3l"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t284" href="#t284">284</a></span><span class="t">        <span class="str">"attn-only-3l-new"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t285" href="#t285">285</a></span><span class="t">        <span class="str">"attn-only-3l-c4-code"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t286" href="#t286">286</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t287" href="#t287">287</a></span><span class="t">    <span class="str">"NeelNanda/Attn_Only_4L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t288" href="#t288">288</a></span><span class="t">        <span class="str">"attn-only-4l"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t289" href="#t289">289</a></span><span class="t">        <span class="str">"attn-only-4l-new"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t290" href="#t290">290</a></span><span class="t">        <span class="str">"attn-only-4l-c4-code"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t291" href="#t291">291</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t292" href="#t292">292</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_6L768W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-6l"</span><span class="op">,</span> <span class="str">"solu-6l-new"</span><span class="op">,</span> <span class="str">"solu-6l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t293" href="#t293">293</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_8L1024W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-8l"</span><span class="op">,</span> <span class="str">"solu-8l-new"</span><span class="op">,</span> <span class="str">"solu-8l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t294" href="#t294">294</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_10L1280W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-10l"</span><span class="op">,</span> <span class="str">"solu-10l-new"</span><span class="op">,</span> <span class="str">"solu-10l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t295" href="#t295">295</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_12L1536W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-12l"</span><span class="op">,</span> <span class="str">"solu-12l-new"</span><span class="op">,</span> <span class="str">"solu-12l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t296" href="#t296">296</a></span><span class="t">    <span class="str">"NeelNanda/Attn-Only-2L512W-Shortformer-6B-big-lr"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t297" href="#t297">297</a></span><span class="t">        <span class="str">"attn-only-2l-demo"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t298" href="#t298">298</a></span><span class="t">        <span class="str">"attn-only-2l-shortformer-6b-big-lr"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t299" href="#t299">299</a></span><span class="t">        <span class="str">"attn-only-2l-induction-demo"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t300" href="#t300">300</a></span><span class="t">        <span class="str">"attn-only-demo"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t301" href="#t301">301</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t302" href="#t302">302</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_1L512W_Wiki_Finetune"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t303" href="#t303">303</a></span><span class="t">        <span class="str">"solu-1l-wiki"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t304" href="#t304">304</a></span><span class="t">        <span class="str">"solu-1l-wiki-finetune"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t305" href="#t305">305</a></span><span class="t">        <span class="str">"solu-1l-finetune"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t306" href="#t306">306</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t307" href="#t307">307</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_4L512W_Wiki_Finetune"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t308" href="#t308">308</a></span><span class="t">        <span class="str">"solu-4l-wiki"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t309" href="#t309">309</a></span><span class="t">        <span class="str">"solu-4l-wiki-finetune"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t310" href="#t310">310</a></span><span class="t">        <span class="str">"solu-4l-finetune"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t311" href="#t311">311</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t312" href="#t312">312</a></span><span class="t">    <span class="str">"EleutherAI/pythia-14m"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t313" href="#t313">313</a></span><span class="t">        <span class="str">"pythia-14m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t314" href="#t314">314</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t315" href="#t315">315</a></span><span class="t">    <span class="str">"EleutherAI/pythia-31m"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t316" href="#t316">316</a></span><span class="t">        <span class="str">"pythia-31m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t317" href="#t317">317</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t318" href="#t318">318</a></span><span class="t">    <span class="str">"EleutherAI/pythia-70m"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t319" href="#t319">319</a></span><span class="t">        <span class="str">"pythia-70m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t320" href="#t320">320</a></span><span class="t">        <span class="str">"pythia"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t321" href="#t321">321</a></span><span class="t">        <span class="str">"EleutherAI/pythia-19m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t322" href="#t322">322</a></span><span class="t">        <span class="str">"pythia-19m"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t323" href="#t323">323</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t324" href="#t324">324</a></span><span class="t">    <span class="str">"EleutherAI/pythia-160m"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t325" href="#t325">325</a></span><span class="t">        <span class="str">"pythia-160m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t326" href="#t326">326</a></span><span class="t">        <span class="str">"EleutherAI/pythia-125m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t327" href="#t327">327</a></span><span class="t">        <span class="str">"pythia-125m"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t328" href="#t328">328</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t329" href="#t329">329</a></span><span class="t">    <span class="str">"EleutherAI/pythia-410m"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t330" href="#t330">330</a></span><span class="t">        <span class="str">"pythia-410m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t331" href="#t331">331</a></span><span class="t">        <span class="str">"EleutherAI/pythia-350m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t332" href="#t332">332</a></span><span class="t">        <span class="str">"pythia-350m"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t333" href="#t333">333</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t334" href="#t334">334</a></span><span class="t">    <span class="str">"EleutherAI/pythia-1b"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t335" href="#t335">335</a></span><span class="t">        <span class="str">"pythia-1b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t336" href="#t336">336</a></span><span class="t">        <span class="str">"EleutherAI/pythia-800m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t337" href="#t337">337</a></span><span class="t">        <span class="str">"pythia-800m"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t338" href="#t338">338</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t339" href="#t339">339</a></span><span class="t">    <span class="str">"EleutherAI/pythia-1.4b"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t340" href="#t340">340</a></span><span class="t">        <span class="str">"pythia-1.4b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t341" href="#t341">341</a></span><span class="t">        <span class="str">"EleutherAI/pythia-1.3b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t342" href="#t342">342</a></span><span class="t">        <span class="str">"pythia-1.3b"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t343" href="#t343">343</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t344" href="#t344">344</a></span><span class="t">    <span class="str">"EleutherAI/pythia-2.8b"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t345" href="#t345">345</a></span><span class="t">        <span class="str">"pythia-2.8b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t346" href="#t346">346</a></span><span class="t">        <span class="str">"EleutherAI/pythia-2.7b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t347" href="#t347">347</a></span><span class="t">        <span class="str">"pythia-2.7b"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t348" href="#t348">348</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t349" href="#t349">349</a></span><span class="t">    <span class="str">"EleutherAI/pythia-6.9b"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t350" href="#t350">350</a></span><span class="t">        <span class="str">"pythia-6.9b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t351" href="#t351">351</a></span><span class="t">        <span class="str">"EleutherAI/pythia-6.7b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t352" href="#t352">352</a></span><span class="t">        <span class="str">"pythia-6.7b"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t353" href="#t353">353</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t354" href="#t354">354</a></span><span class="t">    <span class="str">"EleutherAI/pythia-12b"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t355" href="#t355">355</a></span><span class="t">        <span class="str">"pythia-12b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t356" href="#t356">356</a></span><span class="t">        <span class="str">"EleutherAI/pythia-13b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t357" href="#t357">357</a></span><span class="t">        <span class="str">"pythia-13b"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t358" href="#t358">358</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t359" href="#t359">359</a></span><span class="t">    <span class="str">"EleutherAI/pythia-70m-deduped"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t360" href="#t360">360</a></span><span class="t">        <span class="str">"pythia-70m-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t361" href="#t361">361</a></span><span class="t">        <span class="str">"EleutherAI/pythia-19m-deduped"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t362" href="#t362">362</a></span><span class="t">        <span class="str">"pythia-19m-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t363" href="#t363">363</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t364" href="#t364">364</a></span><span class="t">    <span class="str">"EleutherAI/pythia-160m-deduped"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t365" href="#t365">365</a></span><span class="t">        <span class="str">"pythia-160m-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t366" href="#t366">366</a></span><span class="t">        <span class="str">"EleutherAI/pythia-125m-deduped"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t367" href="#t367">367</a></span><span class="t">        <span class="str">"pythia-125m-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t368" href="#t368">368</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t369" href="#t369">369</a></span><span class="t">    <span class="str">"EleutherAI/pythia-410m-deduped"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t370" href="#t370">370</a></span><span class="t">        <span class="str">"pythia-410m-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t371" href="#t371">371</a></span><span class="t">        <span class="str">"EleutherAI/pythia-350m-deduped"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t372" href="#t372">372</a></span><span class="t">        <span class="str">"pythia-350m-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t373" href="#t373">373</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t374" href="#t374">374</a></span><span class="t">    <span class="str">"EleutherAI/pythia-1b-deduped"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t375" href="#t375">375</a></span><span class="t">        <span class="str">"pythia-1b-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t376" href="#t376">376</a></span><span class="t">        <span class="str">"EleutherAI/pythia-800m-deduped"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t377" href="#t377">377</a></span><span class="t">        <span class="str">"pythia-800m-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t378" href="#t378">378</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t379" href="#t379">379</a></span><span class="t">    <span class="str">"EleutherAI/pythia-1.4b-deduped"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t380" href="#t380">380</a></span><span class="t">        <span class="str">"pythia-1.4b-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t381" href="#t381">381</a></span><span class="t">        <span class="str">"EleutherAI/pythia-1.3b-deduped"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t382" href="#t382">382</a></span><span class="t">        <span class="str">"pythia-1.3b-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t383" href="#t383">383</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t384" href="#t384">384</a></span><span class="t">    <span class="str">"EleutherAI/pythia-2.8b-deduped"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t385" href="#t385">385</a></span><span class="t">        <span class="str">"pythia-2.8b-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t386" href="#t386">386</a></span><span class="t">        <span class="str">"EleutherAI/pythia-2.7b-deduped"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t387" href="#t387">387</a></span><span class="t">        <span class="str">"pythia-2.7b-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t388" href="#t388">388</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t389" href="#t389">389</a></span><span class="t">    <span class="str">"EleutherAI/pythia-6.9b-deduped"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t390" href="#t390">390</a></span><span class="t">        <span class="str">"pythia-6.9b-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t391" href="#t391">391</a></span><span class="t">        <span class="str">"EleutherAI/pythia-6.7b-deduped"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t392" href="#t392">392</a></span><span class="t">        <span class="str">"pythia-6.7b-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t393" href="#t393">393</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t394" href="#t394">394</a></span><span class="t">    <span class="str">"EleutherAI/pythia-12b-deduped"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t395" href="#t395">395</a></span><span class="t">        <span class="str">"pythia-12b-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t396" href="#t396">396</a></span><span class="t">        <span class="str">"EleutherAI/pythia-13b-deduped"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t397" href="#t397">397</a></span><span class="t">        <span class="str">"pythia-13b-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t398" href="#t398">398</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t399" href="#t399">399</a></span><span class="t">    <span class="str">"EleutherAI/pythia-70m-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t400" href="#t400">400</a></span><span class="t">        <span class="str">"pythia-70m-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t401" href="#t401">401</a></span><span class="t">        <span class="str">"pythia-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t402" href="#t402">402</a></span><span class="t">        <span class="str">"EleutherAI/pythia-19m-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t403" href="#t403">403</a></span><span class="t">        <span class="str">"pythia-19m-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t404" href="#t404">404</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t405" href="#t405">405</a></span><span class="t">    <span class="str">"EleutherAI/pythia-160m-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t406" href="#t406">406</a></span><span class="t">        <span class="str">"pythia-160m-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t407" href="#t407">407</a></span><span class="t">        <span class="str">"EleutherAI/pythia-125m-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t408" href="#t408">408</a></span><span class="t">        <span class="str">"pythia-125m-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t409" href="#t409">409</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t410" href="#t410">410</a></span><span class="t">    <span class="str">"EleutherAI/pythia-410m-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t411" href="#t411">411</a></span><span class="t">        <span class="str">"pythia-410m-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t412" href="#t412">412</a></span><span class="t">        <span class="str">"EleutherAI/pythia-350m-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t413" href="#t413">413</a></span><span class="t">        <span class="str">"pythia-350m-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t414" href="#t414">414</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t415" href="#t415">415</a></span><span class="t">    <span class="str">"EleutherAI/pythia-1b-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t416" href="#t416">416</a></span><span class="t">        <span class="str">"pythia-1b-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t417" href="#t417">417</a></span><span class="t">        <span class="str">"EleutherAI/pythia-800m-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t418" href="#t418">418</a></span><span class="t">        <span class="str">"pythia-800m-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t419" href="#t419">419</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t420" href="#t420">420</a></span><span class="t">    <span class="str">"EleutherAI/pythia-1.4b-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t421" href="#t421">421</a></span><span class="t">        <span class="str">"pythia-1.4b-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t422" href="#t422">422</a></span><span class="t">        <span class="str">"EleutherAI/pythia-1.3b-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t423" href="#t423">423</a></span><span class="t">        <span class="str">"pythia-1.3b-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t424" href="#t424">424</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t425" href="#t425">425</a></span><span class="t">    <span class="str">"EleutherAI/pythia-2.8b-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t426" href="#t426">426</a></span><span class="t">        <span class="str">"pythia-2.8b-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t427" href="#t427">427</a></span><span class="t">        <span class="str">"EleutherAI/pythia-2.7b-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t428" href="#t428">428</a></span><span class="t">        <span class="str">"pythia-2.7b-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t429" href="#t429">429</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t430" href="#t430">430</a></span><span class="t">    <span class="str">"EleutherAI/pythia-6.9b-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t431" href="#t431">431</a></span><span class="t">        <span class="str">"pythia-6.9b-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t432" href="#t432">432</a></span><span class="t">        <span class="str">"EleutherAI/pythia-6.7b-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t433" href="#t433">433</a></span><span class="t">        <span class="str">"pythia-6.7b-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t434" href="#t434">434</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t435" href="#t435">435</a></span><span class="t">    <span class="str">"EleutherAI/pythia-12b-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t436" href="#t436">436</a></span><span class="t">        <span class="str">"pythia-12b-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t437" href="#t437">437</a></span><span class="t">        <span class="str">"EleutherAI/pythia-13b-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t438" href="#t438">438</a></span><span class="t">        <span class="str">"pythia-13b-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t439" href="#t439">439</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t440" href="#t440">440</a></span><span class="t">    <span class="str">"EleutherAI/pythia-70m-deduped-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t441" href="#t441">441</a></span><span class="t">        <span class="str">"pythia-70m-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t442" href="#t442">442</a></span><span class="t">        <span class="str">"EleutherAI/pythia-19m-deduped-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t443" href="#t443">443</a></span><span class="t">        <span class="str">"pythia-19m-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t444" href="#t444">444</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t445" href="#t445">445</a></span><span class="t">    <span class="str">"EleutherAI/pythia-160m-deduped-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t446" href="#t446">446</a></span><span class="t">        <span class="str">"pythia-160m-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t447" href="#t447">447</a></span><span class="t">        <span class="str">"EleutherAI/pythia-125m-deduped-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t448" href="#t448">448</a></span><span class="t">        <span class="str">"pythia-125m-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t449" href="#t449">449</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t450" href="#t450">450</a></span><span class="t">    <span class="str">"EleutherAI/pythia-410m-deduped-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t451" href="#t451">451</a></span><span class="t">        <span class="str">"pythia-410m-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t452" href="#t452">452</a></span><span class="t">        <span class="str">"EleutherAI/pythia-350m-deduped-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t453" href="#t453">453</a></span><span class="t">        <span class="str">"pythia-350m-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t454" href="#t454">454</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t455" href="#t455">455</a></span><span class="t">    <span class="str">"EleutherAI/pythia-1b-deduped-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t456" href="#t456">456</a></span><span class="t">        <span class="str">"pythia-1b-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t457" href="#t457">457</a></span><span class="t">        <span class="str">"EleutherAI/pythia-800m-deduped-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t458" href="#t458">458</a></span><span class="t">        <span class="str">"pythia-800m-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t459" href="#t459">459</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t460" href="#t460">460</a></span><span class="t">    <span class="str">"EleutherAI/pythia-1.4b-deduped-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t461" href="#t461">461</a></span><span class="t">        <span class="str">"pythia-1.4b-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t462" href="#t462">462</a></span><span class="t">        <span class="str">"EleutherAI/pythia-1.3b-deduped-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t463" href="#t463">463</a></span><span class="t">        <span class="str">"pythia-1.3b-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t464" href="#t464">464</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t465" href="#t465">465</a></span><span class="t">    <span class="str">"EleutherAI/pythia-2.8b-deduped-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t466" href="#t466">466</a></span><span class="t">        <span class="str">"pythia-2.8b-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t467" href="#t467">467</a></span><span class="t">        <span class="str">"EleutherAI/pythia-2.7b-deduped-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t468" href="#t468">468</a></span><span class="t">        <span class="str">"pythia-2.7b-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t469" href="#t469">469</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t470" href="#t470">470</a></span><span class="t">    <span class="str">"EleutherAI/pythia-6.9b-deduped-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t471" href="#t471">471</a></span><span class="t">        <span class="str">"pythia-6.9b-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t472" href="#t472">472</a></span><span class="t">        <span class="str">"EleutherAI/pythia-6.7b-deduped-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t473" href="#t473">473</a></span><span class="t">        <span class="str">"pythia-6.7b-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t474" href="#t474">474</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t475" href="#t475">475</a></span><span class="t">    <span class="str">"EleutherAI/pythia-12b-deduped-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t476" href="#t476">476</a></span><span class="t">        <span class="str">"pythia-12b-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t477" href="#t477">477</a></span><span class="t">        <span class="str">"EleutherAI/pythia-13b-deduped-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t478" href="#t478">478</a></span><span class="t">        <span class="str">"pythia-13b-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t479" href="#t479">479</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t480" href="#t480">480</a></span><span class="t">    <span class="str">"EleutherAI/pythia-160m-seed1"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t481" href="#t481">481</a></span><span class="t">        <span class="str">"pythia-160m-seed1"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t482" href="#t482">482</a></span><span class="t">        <span class="str">"EleutherAI/pythia-125m-seed1"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t483" href="#t483">483</a></span><span class="t">        <span class="str">"pythia-125m-seed1"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t484" href="#t484">484</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t485" href="#t485">485</a></span><span class="t">    <span class="str">"EleutherAI/pythia-160m-seed2"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t486" href="#t486">486</a></span><span class="t">        <span class="str">"pythia-160m-seed2"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t487" href="#t487">487</a></span><span class="t">        <span class="str">"EleutherAI/pythia-125m-seed2"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t488" href="#t488">488</a></span><span class="t">        <span class="str">"pythia-125m-seed2"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t489" href="#t489">489</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t490" href="#t490">490</a></span><span class="t">    <span class="str">"EleutherAI/pythia-160m-seed3"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t491" href="#t491">491</a></span><span class="t">        <span class="str">"pythia-160m-seed3"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t492" href="#t492">492</a></span><span class="t">        <span class="str">"EleutherAI/pythia-125m-seed3"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t493" href="#t493">493</a></span><span class="t">        <span class="str">"pythia-125m-seed3"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t494" href="#t494">494</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t495" href="#t495">495</a></span><span class="t">    <span class="str">"gpt2"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gpt2-small"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t496" href="#t496">496</a></span><span class="t">    <span class="str">"distilgpt2"</span><span class="op">:</span> <span class="op">[</span><span class="str">"distillgpt2"</span><span class="op">,</span> <span class="str">"distill-gpt2"</span><span class="op">,</span> <span class="str">"distil-gpt2"</span><span class="op">,</span> <span class="str">"gpt2-xs"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t497" href="#t497">497</a></span><span class="t">    <span class="str">"facebook/opt-125m"</span><span class="op">:</span> <span class="op">[</span><span class="str">"opt-125m"</span><span class="op">,</span> <span class="str">"opt-small"</span><span class="op">,</span> <span class="str">"opt"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t498" href="#t498">498</a></span><span class="t">    <span class="str">"facebook/opt-1.3b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"opt-1.3b"</span><span class="op">,</span> <span class="str">"opt-medium"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t499" href="#t499">499</a></span><span class="t">    <span class="str">"facebook/opt-2.7b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"opt-2.7b"</span><span class="op">,</span> <span class="str">"opt-large"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t500" href="#t500">500</a></span><span class="t">    <span class="str">"facebook/opt-6.7b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"opt-6.7b"</span><span class="op">,</span> <span class="str">"opt-xl"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t501" href="#t501">501</a></span><span class="t">    <span class="str">"facebook/opt-13b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"opt-13b"</span><span class="op">,</span> <span class="str">"opt-xxl"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t502" href="#t502">502</a></span><span class="t">    <span class="str">"facebook/opt-30b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"opt-30b"</span><span class="op">,</span> <span class="str">"opt-xxxl"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t503" href="#t503">503</a></span><span class="t">    <span class="str">"facebook/opt-66b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"opt-66b"</span><span class="op">,</span> <span class="str">"opt-xxxxl"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t504" href="#t504">504</a></span><span class="t">    <span class="str">"EleutherAI/gpt-neo-125M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gpt-neo-125M"</span><span class="op">,</span> <span class="str">"gpt-neo-small"</span><span class="op">,</span> <span class="str">"neo-small"</span><span class="op">,</span> <span class="str">"neo"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t505" href="#t505">505</a></span><span class="t">    <span class="str">"EleutherAI/gpt-neo-1.3B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gpt-neo-1.3B"</span><span class="op">,</span> <span class="str">"gpt-neo-medium"</span><span class="op">,</span> <span class="str">"neo-medium"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t506" href="#t506">506</a></span><span class="t">    <span class="str">"EleutherAI/gpt-neo-2.7B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gpt-neo-2.7B"</span><span class="op">,</span> <span class="str">"gpt-neo-large"</span><span class="op">,</span> <span class="str">"neo-large"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t507" href="#t507">507</a></span><span class="t">    <span class="str">"EleutherAI/gpt-j-6B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gpt-j-6B"</span><span class="op">,</span> <span class="str">"gpt-j"</span><span class="op">,</span> <span class="str">"gptj"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t508" href="#t508">508</a></span><span class="t">    <span class="str">"EleutherAI/gpt-neox-20b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gpt-neox-20b"</span><span class="op">,</span> <span class="str">"gpt-neox"</span><span class="op">,</span> <span class="str">"neox"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t509" href="#t509">509</a></span><span class="t">    <span class="str">"stanford-crfm/alias-gpt2-small-x21"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t510" href="#t510">510</a></span><span class="t">        <span class="str">"stanford-gpt2-small-a"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t511" href="#t511">511</a></span><span class="t">        <span class="str">"alias-gpt2-small-x21"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t512" href="#t512">512</a></span><span class="t">        <span class="str">"gpt2-mistral-small-a"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t513" href="#t513">513</a></span><span class="t">        <span class="str">"gpt2-stanford-small-a"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t514" href="#t514">514</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t515" href="#t515">515</a></span><span class="t">    <span class="str">"stanford-crfm/battlestar-gpt2-small-x49"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t516" href="#t516">516</a></span><span class="t">        <span class="str">"stanford-gpt2-small-b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t517" href="#t517">517</a></span><span class="t">        <span class="str">"battlestar-gpt2-small-x49"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t518" href="#t518">518</a></span><span class="t">        <span class="str">"gpt2-mistral-small-b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t154" href="#t154">154</a></span><span class="t">    <span class="str">"meta-llama/Llama-3.1-70B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t155" href="#t155">155</a></span><span class="t">    <span class="str">"meta-llama/Llama-3.1-8B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t156" href="#t156">156</a></span><span class="t">    <span class="str">"meta-llama/Llama-3.1-8B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t157" href="#t157">157</a></span><span class="t">    <span class="str">"meta-llama/Llama-3.1-70B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t158" href="#t158">158</a></span><span class="t">    <span class="str">"meta-llama/Llama-3.2-1B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t159" href="#t159">159</a></span><span class="t">    <span class="str">"meta-llama/Llama-3.2-3B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t160" href="#t160">160</a></span><span class="t">    <span class="str">"meta-llama/Llama-3.2-1B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t161" href="#t161">161</a></span><span class="t">    <span class="str">"meta-llama/Llama-3.2-3B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t162" href="#t162">162</a></span><span class="t">    <span class="str">"meta-llama/Llama-3.3-70B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t163" href="#t163">163</a></span><span class="t">    <span class="str">"Baidicoot/Othello-GPT-Transformer-Lens"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t164" href="#t164">164</a></span><span class="t">    <span class="str">"bert-base-cased"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t165" href="#t165">165</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-1M"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t166" href="#t166">166</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-3M"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t167" href="#t167">167</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-8M"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t168" href="#t168">168</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-28M"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t169" href="#t169">169</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-33M"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t170" href="#t170">170</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instruct-1M"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t171" href="#t171">171</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instruct-3M"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t172" href="#t172">172</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instruct-8M"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t173" href="#t173">173</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instruct-28M"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t174" href="#t174">174</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instruct-33M"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t175" href="#t175">175</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-1Layer-21M"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t176" href="#t176">176</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-2Layers-33M"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t177" href="#t177">177</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instuct-1Layer-21M"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t178" href="#t178">178</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instruct-2Layers-33M"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t179" href="#t179">179</a></span><span class="t">    <span class="str">"stabilityai/stablelm-base-alpha-3b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t180" href="#t180">180</a></span><span class="t">    <span class="str">"stabilityai/stablelm-base-alpha-7b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t181" href="#t181">181</a></span><span class="t">    <span class="str">"stabilityai/stablelm-tuned-alpha-3b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t182" href="#t182">182</a></span><span class="t">    <span class="str">"stabilityai/stablelm-tuned-alpha-7b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t183" href="#t183">183</a></span><span class="t">    <span class="str">"mistralai/Mistral-7B-v0.1"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t184" href="#t184">184</a></span><span class="t">    <span class="str">"mistralai/Mistral-7B-Instruct-v0.1"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t185" href="#t185">185</a></span><span class="t">    <span class="str">"mistralai/Mistral-Nemo-Base-2407"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t186" href="#t186">186</a></span><span class="t">    <span class="str">"mistralai/Mixtral-8x7B-v0.1"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t187" href="#t187">187</a></span><span class="t">    <span class="str">"mistralai/Mixtral-8x7B-Instruct-v0.1"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t188" href="#t188">188</a></span><span class="t">    <span class="str">"bigscience/bloom-560m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t189" href="#t189">189</a></span><span class="t">    <span class="str">"bigscience/bloom-1b1"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t190" href="#t190">190</a></span><span class="t">    <span class="str">"bigscience/bloom-1b7"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t191" href="#t191">191</a></span><span class="t">    <span class="str">"bigscience/bloom-3b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t192" href="#t192">192</a></span><span class="t">    <span class="str">"bigscience/bloom-7b1"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t193" href="#t193">193</a></span><span class="t">    <span class="str">"bigcode/santacoder"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t194" href="#t194">194</a></span><span class="t">    <span class="str">"Qwen/Qwen-1_8B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t195" href="#t195">195</a></span><span class="t">    <span class="str">"Qwen/Qwen-7B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t196" href="#t196">196</a></span><span class="t">    <span class="str">"Qwen/Qwen-14B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t197" href="#t197">197</a></span><span class="t">    <span class="str">"Qwen/Qwen-1_8B-Chat"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t198" href="#t198">198</a></span><span class="t">    <span class="str">"Qwen/Qwen-7B-Chat"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t199" href="#t199">199</a></span><span class="t">    <span class="str">"Qwen/Qwen-14B-Chat"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t200" href="#t200">200</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-0.5B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t201" href="#t201">201</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-0.5B-Chat"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t202" href="#t202">202</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-1.8B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t203" href="#t203">203</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-1.8B-Chat"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t204" href="#t204">204</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-4B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t205" href="#t205">205</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-4B-Chat"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t206" href="#t206">206</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-7B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t207" href="#t207">207</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-7B-Chat"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t208" href="#t208">208</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-14B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t209" href="#t209">209</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-14B-Chat"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t210" href="#t210">210</a></span><span class="t">    <span class="str">"Qwen/Qwen2-0.5B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t211" href="#t211">211</a></span><span class="t">    <span class="str">"Qwen/Qwen2-0.5B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t212" href="#t212">212</a></span><span class="t">    <span class="str">"Qwen/Qwen2-1.5B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t213" href="#t213">213</a></span><span class="t">    <span class="str">"Qwen/Qwen2-1.5B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t214" href="#t214">214</a></span><span class="t">    <span class="str">"Qwen/Qwen2-7B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t215" href="#t215">215</a></span><span class="t">    <span class="str">"Qwen/Qwen2-7B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t216" href="#t216">216</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-0.5B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t217" href="#t217">217</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-0.5B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t218" href="#t218">218</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-1.5B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t219" href="#t219">219</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-1.5B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t220" href="#t220">220</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-3B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t221" href="#t221">221</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-3B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t222" href="#t222">222</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-7B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t223" href="#t223">223</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-7B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t224" href="#t224">224</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-14B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t225" href="#t225">225</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-14B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t226" href="#t226">226</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-32B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t227" href="#t227">227</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-32B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t228" href="#t228">228</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-72B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t229" href="#t229">229</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-72B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t230" href="#t230">230</a></span><span class="t">    <span class="str">"Qwen/QwQ-32B-Preview"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t231" href="#t231">231</a></span><span class="t">    <span class="str">"microsoft/phi-1"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t232" href="#t232">232</a></span><span class="t">    <span class="str">"microsoft/phi-1_5"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t233" href="#t233">233</a></span><span class="t">    <span class="str">"microsoft/phi-2"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t234" href="#t234">234</a></span><span class="t">    <span class="str">"microsoft/Phi-3-mini-4k-instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t235" href="#t235">235</a></span><span class="t">    <span class="str">"google/gemma-2b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t236" href="#t236">236</a></span><span class="t">    <span class="str">"google/gemma-7b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t237" href="#t237">237</a></span><span class="t">    <span class="str">"google/gemma-2b-it"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t238" href="#t238">238</a></span><span class="t">    <span class="str">"google/gemma-7b-it"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t239" href="#t239">239</a></span><span class="t">    <span class="str">"google/gemma-2-2b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t240" href="#t240">240</a></span><span class="t">    <span class="str">"google/gemma-2-2b-it"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t241" href="#t241">241</a></span><span class="t">    <span class="str">"google/gemma-2-9b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t242" href="#t242">242</a></span><span class="t">    <span class="str">"google/gemma-2-9b-it"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t243" href="#t243">243</a></span><span class="t">    <span class="str">"google/gemma-2-27b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t244" href="#t244">244</a></span><span class="t">    <span class="str">"google/gemma-2-27b-it"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t245" href="#t245">245</a></span><span class="t">    <span class="str">"01-ai/Yi-6B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t246" href="#t246">246</a></span><span class="t">    <span class="str">"01-ai/Yi-34B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t247" href="#t247">247</a></span><span class="t">    <span class="str">"01-ai/Yi-6B-Chat"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t248" href="#t248">248</a></span><span class="t">    <span class="str">"01-ai/Yi-34B-Chat"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t249" href="#t249">249</a></span><span class="t">    <span class="str">"google-t5/t5-small"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t250" href="#t250">250</a></span><span class="t">    <span class="str">"google-t5/t5-base"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t251" href="#t251">251</a></span><span class="t">    <span class="str">"google-t5/t5-large"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t252" href="#t252">252</a></span><span class="t">    <span class="str">"ai-forever/mGPT"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t253" href="#t253">253</a></span><span class="t"><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t254" href="#t254">254</a></span><span class="t"><span class="str">"""Official model names for models on HuggingFace."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t255" href="#t255">255</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t256" href="#t256">256</a></span><span class="t"><span class="com"># Model Aliases:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t257" href="#t257">257</a></span><span class="t"><span class="nam">MODEL_ALIASES</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t258" href="#t258">258</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_1L_v9_old"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-1l-pile"</span><span class="op">,</span> <span class="str">"solu-1l-old"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t259" href="#t259">259</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_2L_v10_old"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-2l-pile"</span><span class="op">,</span> <span class="str">"solu-2l-old"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t260" href="#t260">260</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_4L_v11_old"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-4l-pile"</span><span class="op">,</span> <span class="str">"solu-4l-old"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t261" href="#t261">261</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_6L_v13_old"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-6l-pile"</span><span class="op">,</span> <span class="str">"solu-6l-old"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t262" href="#t262">262</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_8L_v21_old"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-8l-pile"</span><span class="op">,</span> <span class="str">"solu-8l-old"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t263" href="#t263">263</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_10L_v22_old"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-10l-pile"</span><span class="op">,</span> <span class="str">"solu-10l-old"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t264" href="#t264">264</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_12L_v23_old"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-12l-pile"</span><span class="op">,</span> <span class="str">"solu-12l-old"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t265" href="#t265">265</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_1L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-1l"</span><span class="op">,</span> <span class="str">"solu-1l-new"</span><span class="op">,</span> <span class="str">"solu-1l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t266" href="#t266">266</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_2L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-2l"</span><span class="op">,</span> <span class="str">"solu-2l-new"</span><span class="op">,</span> <span class="str">"solu-2l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t267" href="#t267">267</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_3L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-3l"</span><span class="op">,</span> <span class="str">"solu-3l-new"</span><span class="op">,</span> <span class="str">"solu-3l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t268" href="#t268">268</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_4L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-4l"</span><span class="op">,</span> <span class="str">"solu-4l-new"</span><span class="op">,</span> <span class="str">"solu-4l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t269" href="#t269">269</a></span><span class="t">    <span class="str">"NeelNanda/GELU_1L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gelu-1l"</span><span class="op">,</span> <span class="str">"gelu-1l-new"</span><span class="op">,</span> <span class="str">"gelu-1l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t270" href="#t270">270</a></span><span class="t">    <span class="str">"NeelNanda/GELU_2L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gelu-2l"</span><span class="op">,</span> <span class="str">"gelu-2l-new"</span><span class="op">,</span> <span class="str">"gelu-2l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t271" href="#t271">271</a></span><span class="t">    <span class="str">"NeelNanda/GELU_3L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gelu-3l"</span><span class="op">,</span> <span class="str">"gelu-3l-new"</span><span class="op">,</span> <span class="str">"gelu-3l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t272" href="#t272">272</a></span><span class="t">    <span class="str">"NeelNanda/GELU_4L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gelu-4l"</span><span class="op">,</span> <span class="str">"gelu-4l-new"</span><span class="op">,</span> <span class="str">"gelu-4l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t273" href="#t273">273</a></span><span class="t">    <span class="str">"NeelNanda/Attn_Only_1L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t274" href="#t274">274</a></span><span class="t">        <span class="str">"attn-only-1l"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t275" href="#t275">275</a></span><span class="t">        <span class="str">"attn-only-1l-new"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t276" href="#t276">276</a></span><span class="t">        <span class="str">"attn-only-1l-c4-code"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t277" href="#t277">277</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t278" href="#t278">278</a></span><span class="t">    <span class="str">"NeelNanda/Attn_Only_2L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t279" href="#t279">279</a></span><span class="t">        <span class="str">"attn-only-2l"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t280" href="#t280">280</a></span><span class="t">        <span class="str">"attn-only-2l-new"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t281" href="#t281">281</a></span><span class="t">        <span class="str">"attn-only-2l-c4-code"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t282" href="#t282">282</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t283" href="#t283">283</a></span><span class="t">    <span class="str">"NeelNanda/Attn_Only_3L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t284" href="#t284">284</a></span><span class="t">        <span class="str">"attn-only-3l"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t285" href="#t285">285</a></span><span class="t">        <span class="str">"attn-only-3l-new"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t286" href="#t286">286</a></span><span class="t">        <span class="str">"attn-only-3l-c4-code"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t287" href="#t287">287</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t288" href="#t288">288</a></span><span class="t">    <span class="str">"NeelNanda/Attn_Only_4L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t289" href="#t289">289</a></span><span class="t">        <span class="str">"attn-only-4l"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t290" href="#t290">290</a></span><span class="t">        <span class="str">"attn-only-4l-new"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t291" href="#t291">291</a></span><span class="t">        <span class="str">"attn-only-4l-c4-code"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t292" href="#t292">292</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t293" href="#t293">293</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_6L768W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-6l"</span><span class="op">,</span> <span class="str">"solu-6l-new"</span><span class="op">,</span> <span class="str">"solu-6l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t294" href="#t294">294</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_8L1024W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-8l"</span><span class="op">,</span> <span class="str">"solu-8l-new"</span><span class="op">,</span> <span class="str">"solu-8l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t295" href="#t295">295</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_10L1280W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-10l"</span><span class="op">,</span> <span class="str">"solu-10l-new"</span><span class="op">,</span> <span class="str">"solu-10l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t296" href="#t296">296</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_12L1536W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-12l"</span><span class="op">,</span> <span class="str">"solu-12l-new"</span><span class="op">,</span> <span class="str">"solu-12l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t297" href="#t297">297</a></span><span class="t">    <span class="str">"NeelNanda/Attn-Only-2L512W-Shortformer-6B-big-lr"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t298" href="#t298">298</a></span><span class="t">        <span class="str">"attn-only-2l-demo"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t299" href="#t299">299</a></span><span class="t">        <span class="str">"attn-only-2l-shortformer-6b-big-lr"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t300" href="#t300">300</a></span><span class="t">        <span class="str">"attn-only-2l-induction-demo"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t301" href="#t301">301</a></span><span class="t">        <span class="str">"attn-only-demo"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t302" href="#t302">302</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t303" href="#t303">303</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_1L512W_Wiki_Finetune"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t304" href="#t304">304</a></span><span class="t">        <span class="str">"solu-1l-wiki"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t305" href="#t305">305</a></span><span class="t">        <span class="str">"solu-1l-wiki-finetune"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t306" href="#t306">306</a></span><span class="t">        <span class="str">"solu-1l-finetune"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t307" href="#t307">307</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t308" href="#t308">308</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_4L512W_Wiki_Finetune"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t309" href="#t309">309</a></span><span class="t">        <span class="str">"solu-4l-wiki"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t310" href="#t310">310</a></span><span class="t">        <span class="str">"solu-4l-wiki-finetune"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t311" href="#t311">311</a></span><span class="t">        <span class="str">"solu-4l-finetune"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t312" href="#t312">312</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t313" href="#t313">313</a></span><span class="t">    <span class="str">"EleutherAI/pythia-14m"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t314" href="#t314">314</a></span><span class="t">        <span class="str">"pythia-14m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t315" href="#t315">315</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t316" href="#t316">316</a></span><span class="t">    <span class="str">"EleutherAI/pythia-31m"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t317" href="#t317">317</a></span><span class="t">        <span class="str">"pythia-31m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t318" href="#t318">318</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t319" href="#t319">319</a></span><span class="t">    <span class="str">"EleutherAI/pythia-70m"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t320" href="#t320">320</a></span><span class="t">        <span class="str">"pythia-70m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t321" href="#t321">321</a></span><span class="t">        <span class="str">"pythia"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t322" href="#t322">322</a></span><span class="t">        <span class="str">"EleutherAI/pythia-19m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t323" href="#t323">323</a></span><span class="t">        <span class="str">"pythia-19m"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t324" href="#t324">324</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t325" href="#t325">325</a></span><span class="t">    <span class="str">"EleutherAI/pythia-160m"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t326" href="#t326">326</a></span><span class="t">        <span class="str">"pythia-160m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t327" href="#t327">327</a></span><span class="t">        <span class="str">"EleutherAI/pythia-125m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t328" href="#t328">328</a></span><span class="t">        <span class="str">"pythia-125m"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t329" href="#t329">329</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t330" href="#t330">330</a></span><span class="t">    <span class="str">"EleutherAI/pythia-410m"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t331" href="#t331">331</a></span><span class="t">        <span class="str">"pythia-410m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t332" href="#t332">332</a></span><span class="t">        <span class="str">"EleutherAI/pythia-350m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t333" href="#t333">333</a></span><span class="t">        <span class="str">"pythia-350m"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t334" href="#t334">334</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t335" href="#t335">335</a></span><span class="t">    <span class="str">"EleutherAI/pythia-1b"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t336" href="#t336">336</a></span><span class="t">        <span class="str">"pythia-1b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t337" href="#t337">337</a></span><span class="t">        <span class="str">"EleutherAI/pythia-800m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t338" href="#t338">338</a></span><span class="t">        <span class="str">"pythia-800m"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t339" href="#t339">339</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t340" href="#t340">340</a></span><span class="t">    <span class="str">"EleutherAI/pythia-1.4b"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t341" href="#t341">341</a></span><span class="t">        <span class="str">"pythia-1.4b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t342" href="#t342">342</a></span><span class="t">        <span class="str">"EleutherAI/pythia-1.3b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t343" href="#t343">343</a></span><span class="t">        <span class="str">"pythia-1.3b"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t344" href="#t344">344</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t345" href="#t345">345</a></span><span class="t">    <span class="str">"EleutherAI/pythia-2.8b"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t346" href="#t346">346</a></span><span class="t">        <span class="str">"pythia-2.8b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t347" href="#t347">347</a></span><span class="t">        <span class="str">"EleutherAI/pythia-2.7b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t348" href="#t348">348</a></span><span class="t">        <span class="str">"pythia-2.7b"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t349" href="#t349">349</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t350" href="#t350">350</a></span><span class="t">    <span class="str">"EleutherAI/pythia-6.9b"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t351" href="#t351">351</a></span><span class="t">        <span class="str">"pythia-6.9b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t352" href="#t352">352</a></span><span class="t">        <span class="str">"EleutherAI/pythia-6.7b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t353" href="#t353">353</a></span><span class="t">        <span class="str">"pythia-6.7b"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t354" href="#t354">354</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t355" href="#t355">355</a></span><span class="t">    <span class="str">"EleutherAI/pythia-12b"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t356" href="#t356">356</a></span><span class="t">        <span class="str">"pythia-12b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t357" href="#t357">357</a></span><span class="t">        <span class="str">"EleutherAI/pythia-13b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t358" href="#t358">358</a></span><span class="t">        <span class="str">"pythia-13b"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t359" href="#t359">359</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t360" href="#t360">360</a></span><span class="t">    <span class="str">"EleutherAI/pythia-70m-deduped"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t361" href="#t361">361</a></span><span class="t">        <span class="str">"pythia-70m-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t362" href="#t362">362</a></span><span class="t">        <span class="str">"EleutherAI/pythia-19m-deduped"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t363" href="#t363">363</a></span><span class="t">        <span class="str">"pythia-19m-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t364" href="#t364">364</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t365" href="#t365">365</a></span><span class="t">    <span class="str">"EleutherAI/pythia-160m-deduped"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t366" href="#t366">366</a></span><span class="t">        <span class="str">"pythia-160m-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t367" href="#t367">367</a></span><span class="t">        <span class="str">"EleutherAI/pythia-125m-deduped"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t368" href="#t368">368</a></span><span class="t">        <span class="str">"pythia-125m-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t369" href="#t369">369</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t370" href="#t370">370</a></span><span class="t">    <span class="str">"EleutherAI/pythia-410m-deduped"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t371" href="#t371">371</a></span><span class="t">        <span class="str">"pythia-410m-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t372" href="#t372">372</a></span><span class="t">        <span class="str">"EleutherAI/pythia-350m-deduped"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t373" href="#t373">373</a></span><span class="t">        <span class="str">"pythia-350m-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t374" href="#t374">374</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t375" href="#t375">375</a></span><span class="t">    <span class="str">"EleutherAI/pythia-1b-deduped"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t376" href="#t376">376</a></span><span class="t">        <span class="str">"pythia-1b-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t377" href="#t377">377</a></span><span class="t">        <span class="str">"EleutherAI/pythia-800m-deduped"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t378" href="#t378">378</a></span><span class="t">        <span class="str">"pythia-800m-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t379" href="#t379">379</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t380" href="#t380">380</a></span><span class="t">    <span class="str">"EleutherAI/pythia-1.4b-deduped"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t381" href="#t381">381</a></span><span class="t">        <span class="str">"pythia-1.4b-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t382" href="#t382">382</a></span><span class="t">        <span class="str">"EleutherAI/pythia-1.3b-deduped"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t383" href="#t383">383</a></span><span class="t">        <span class="str">"pythia-1.3b-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t384" href="#t384">384</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t385" href="#t385">385</a></span><span class="t">    <span class="str">"EleutherAI/pythia-2.8b-deduped"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t386" href="#t386">386</a></span><span class="t">        <span class="str">"pythia-2.8b-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t387" href="#t387">387</a></span><span class="t">        <span class="str">"EleutherAI/pythia-2.7b-deduped"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t388" href="#t388">388</a></span><span class="t">        <span class="str">"pythia-2.7b-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t389" href="#t389">389</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t390" href="#t390">390</a></span><span class="t">    <span class="str">"EleutherAI/pythia-6.9b-deduped"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t391" href="#t391">391</a></span><span class="t">        <span class="str">"pythia-6.9b-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t392" href="#t392">392</a></span><span class="t">        <span class="str">"EleutherAI/pythia-6.7b-deduped"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t393" href="#t393">393</a></span><span class="t">        <span class="str">"pythia-6.7b-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t394" href="#t394">394</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t395" href="#t395">395</a></span><span class="t">    <span class="str">"EleutherAI/pythia-12b-deduped"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t396" href="#t396">396</a></span><span class="t">        <span class="str">"pythia-12b-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t397" href="#t397">397</a></span><span class="t">        <span class="str">"EleutherAI/pythia-13b-deduped"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t398" href="#t398">398</a></span><span class="t">        <span class="str">"pythia-13b-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t399" href="#t399">399</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t400" href="#t400">400</a></span><span class="t">    <span class="str">"EleutherAI/pythia-70m-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t401" href="#t401">401</a></span><span class="t">        <span class="str">"pythia-70m-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t402" href="#t402">402</a></span><span class="t">        <span class="str">"pythia-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t403" href="#t403">403</a></span><span class="t">        <span class="str">"EleutherAI/pythia-19m-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t404" href="#t404">404</a></span><span class="t">        <span class="str">"pythia-19m-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t405" href="#t405">405</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t406" href="#t406">406</a></span><span class="t">    <span class="str">"EleutherAI/pythia-160m-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t407" href="#t407">407</a></span><span class="t">        <span class="str">"pythia-160m-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t408" href="#t408">408</a></span><span class="t">        <span class="str">"EleutherAI/pythia-125m-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t409" href="#t409">409</a></span><span class="t">        <span class="str">"pythia-125m-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t410" href="#t410">410</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t411" href="#t411">411</a></span><span class="t">    <span class="str">"EleutherAI/pythia-410m-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t412" href="#t412">412</a></span><span class="t">        <span class="str">"pythia-410m-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t413" href="#t413">413</a></span><span class="t">        <span class="str">"EleutherAI/pythia-350m-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t414" href="#t414">414</a></span><span class="t">        <span class="str">"pythia-350m-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t415" href="#t415">415</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t416" href="#t416">416</a></span><span class="t">    <span class="str">"EleutherAI/pythia-1b-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t417" href="#t417">417</a></span><span class="t">        <span class="str">"pythia-1b-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t418" href="#t418">418</a></span><span class="t">        <span class="str">"EleutherAI/pythia-800m-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t419" href="#t419">419</a></span><span class="t">        <span class="str">"pythia-800m-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t420" href="#t420">420</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t421" href="#t421">421</a></span><span class="t">    <span class="str">"EleutherAI/pythia-1.4b-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t422" href="#t422">422</a></span><span class="t">        <span class="str">"pythia-1.4b-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t423" href="#t423">423</a></span><span class="t">        <span class="str">"EleutherAI/pythia-1.3b-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t424" href="#t424">424</a></span><span class="t">        <span class="str">"pythia-1.3b-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t425" href="#t425">425</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t426" href="#t426">426</a></span><span class="t">    <span class="str">"EleutherAI/pythia-2.8b-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t427" href="#t427">427</a></span><span class="t">        <span class="str">"pythia-2.8b-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t428" href="#t428">428</a></span><span class="t">        <span class="str">"EleutherAI/pythia-2.7b-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t429" href="#t429">429</a></span><span class="t">        <span class="str">"pythia-2.7b-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t430" href="#t430">430</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t431" href="#t431">431</a></span><span class="t">    <span class="str">"EleutherAI/pythia-6.9b-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t432" href="#t432">432</a></span><span class="t">        <span class="str">"pythia-6.9b-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t433" href="#t433">433</a></span><span class="t">        <span class="str">"EleutherAI/pythia-6.7b-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t434" href="#t434">434</a></span><span class="t">        <span class="str">"pythia-6.7b-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t435" href="#t435">435</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t436" href="#t436">436</a></span><span class="t">    <span class="str">"EleutherAI/pythia-12b-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t437" href="#t437">437</a></span><span class="t">        <span class="str">"pythia-12b-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t438" href="#t438">438</a></span><span class="t">        <span class="str">"EleutherAI/pythia-13b-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t439" href="#t439">439</a></span><span class="t">        <span class="str">"pythia-13b-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t440" href="#t440">440</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t441" href="#t441">441</a></span><span class="t">    <span class="str">"EleutherAI/pythia-70m-deduped-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t442" href="#t442">442</a></span><span class="t">        <span class="str">"pythia-70m-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t443" href="#t443">443</a></span><span class="t">        <span class="str">"EleutherAI/pythia-19m-deduped-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t444" href="#t444">444</a></span><span class="t">        <span class="str">"pythia-19m-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t445" href="#t445">445</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t446" href="#t446">446</a></span><span class="t">    <span class="str">"EleutherAI/pythia-160m-deduped-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t447" href="#t447">447</a></span><span class="t">        <span class="str">"pythia-160m-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t448" href="#t448">448</a></span><span class="t">        <span class="str">"EleutherAI/pythia-125m-deduped-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t449" href="#t449">449</a></span><span class="t">        <span class="str">"pythia-125m-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t450" href="#t450">450</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t451" href="#t451">451</a></span><span class="t">    <span class="str">"EleutherAI/pythia-410m-deduped-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t452" href="#t452">452</a></span><span class="t">        <span class="str">"pythia-410m-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t453" href="#t453">453</a></span><span class="t">        <span class="str">"EleutherAI/pythia-350m-deduped-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t454" href="#t454">454</a></span><span class="t">        <span class="str">"pythia-350m-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t455" href="#t455">455</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t456" href="#t456">456</a></span><span class="t">    <span class="str">"EleutherAI/pythia-1b-deduped-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t457" href="#t457">457</a></span><span class="t">        <span class="str">"pythia-1b-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t458" href="#t458">458</a></span><span class="t">        <span class="str">"EleutherAI/pythia-800m-deduped-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t459" href="#t459">459</a></span><span class="t">        <span class="str">"pythia-800m-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t460" href="#t460">460</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t461" href="#t461">461</a></span><span class="t">    <span class="str">"EleutherAI/pythia-1.4b-deduped-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t462" href="#t462">462</a></span><span class="t">        <span class="str">"pythia-1.4b-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t463" href="#t463">463</a></span><span class="t">        <span class="str">"EleutherAI/pythia-1.3b-deduped-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t464" href="#t464">464</a></span><span class="t">        <span class="str">"pythia-1.3b-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t465" href="#t465">465</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t466" href="#t466">466</a></span><span class="t">    <span class="str">"EleutherAI/pythia-2.8b-deduped-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t467" href="#t467">467</a></span><span class="t">        <span class="str">"pythia-2.8b-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t468" href="#t468">468</a></span><span class="t">        <span class="str">"EleutherAI/pythia-2.7b-deduped-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t469" href="#t469">469</a></span><span class="t">        <span class="str">"pythia-2.7b-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t470" href="#t470">470</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t471" href="#t471">471</a></span><span class="t">    <span class="str">"EleutherAI/pythia-6.9b-deduped-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t472" href="#t472">472</a></span><span class="t">        <span class="str">"pythia-6.9b-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t473" href="#t473">473</a></span><span class="t">        <span class="str">"EleutherAI/pythia-6.7b-deduped-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t474" href="#t474">474</a></span><span class="t">        <span class="str">"pythia-6.7b-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t475" href="#t475">475</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t476" href="#t476">476</a></span><span class="t">    <span class="str">"EleutherAI/pythia-12b-deduped-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t477" href="#t477">477</a></span><span class="t">        <span class="str">"pythia-12b-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t478" href="#t478">478</a></span><span class="t">        <span class="str">"EleutherAI/pythia-13b-deduped-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t479" href="#t479">479</a></span><span class="t">        <span class="str">"pythia-13b-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t480" href="#t480">480</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t481" href="#t481">481</a></span><span class="t">    <span class="str">"EleutherAI/pythia-160m-seed1"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t482" href="#t482">482</a></span><span class="t">        <span class="str">"pythia-160m-seed1"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t483" href="#t483">483</a></span><span class="t">        <span class="str">"EleutherAI/pythia-125m-seed1"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t484" href="#t484">484</a></span><span class="t">        <span class="str">"pythia-125m-seed1"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t485" href="#t485">485</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t486" href="#t486">486</a></span><span class="t">    <span class="str">"EleutherAI/pythia-160m-seed2"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t487" href="#t487">487</a></span><span class="t">        <span class="str">"pythia-160m-seed2"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t488" href="#t488">488</a></span><span class="t">        <span class="str">"EleutherAI/pythia-125m-seed2"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t489" href="#t489">489</a></span><span class="t">        <span class="str">"pythia-125m-seed2"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t490" href="#t490">490</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t491" href="#t491">491</a></span><span class="t">    <span class="str">"EleutherAI/pythia-160m-seed3"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t492" href="#t492">492</a></span><span class="t">        <span class="str">"pythia-160m-seed3"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t493" href="#t493">493</a></span><span class="t">        <span class="str">"EleutherAI/pythia-125m-seed3"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t494" href="#t494">494</a></span><span class="t">        <span class="str">"pythia-125m-seed3"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t495" href="#t495">495</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t496" href="#t496">496</a></span><span class="t">    <span class="str">"gpt2"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gpt2-small"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t497" href="#t497">497</a></span><span class="t">    <span class="str">"distilgpt2"</span><span class="op">:</span> <span class="op">[</span><span class="str">"distillgpt2"</span><span class="op">,</span> <span class="str">"distill-gpt2"</span><span class="op">,</span> <span class="str">"distil-gpt2"</span><span class="op">,</span> <span class="str">"gpt2-xs"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t498" href="#t498">498</a></span><span class="t">    <span class="str">"facebook/opt-125m"</span><span class="op">:</span> <span class="op">[</span><span class="str">"opt-125m"</span><span class="op">,</span> <span class="str">"opt-small"</span><span class="op">,</span> <span class="str">"opt"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t499" href="#t499">499</a></span><span class="t">    <span class="str">"facebook/opt-1.3b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"opt-1.3b"</span><span class="op">,</span> <span class="str">"opt-medium"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t500" href="#t500">500</a></span><span class="t">    <span class="str">"facebook/opt-2.7b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"opt-2.7b"</span><span class="op">,</span> <span class="str">"opt-large"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t501" href="#t501">501</a></span><span class="t">    <span class="str">"facebook/opt-6.7b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"opt-6.7b"</span><span class="op">,</span> <span class="str">"opt-xl"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t502" href="#t502">502</a></span><span class="t">    <span class="str">"facebook/opt-13b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"opt-13b"</span><span class="op">,</span> <span class="str">"opt-xxl"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t503" href="#t503">503</a></span><span class="t">    <span class="str">"facebook/opt-30b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"opt-30b"</span><span class="op">,</span> <span class="str">"opt-xxxl"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t504" href="#t504">504</a></span><span class="t">    <span class="str">"facebook/opt-66b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"opt-66b"</span><span class="op">,</span> <span class="str">"opt-xxxxl"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t505" href="#t505">505</a></span><span class="t">    <span class="str">"EleutherAI/gpt-neo-125M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gpt-neo-125M"</span><span class="op">,</span> <span class="str">"gpt-neo-small"</span><span class="op">,</span> <span class="str">"neo-small"</span><span class="op">,</span> <span class="str">"neo"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t506" href="#t506">506</a></span><span class="t">    <span class="str">"EleutherAI/gpt-neo-1.3B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gpt-neo-1.3B"</span><span class="op">,</span> <span class="str">"gpt-neo-medium"</span><span class="op">,</span> <span class="str">"neo-medium"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t507" href="#t507">507</a></span><span class="t">    <span class="str">"EleutherAI/gpt-neo-2.7B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gpt-neo-2.7B"</span><span class="op">,</span> <span class="str">"gpt-neo-large"</span><span class="op">,</span> <span class="str">"neo-large"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t508" href="#t508">508</a></span><span class="t">    <span class="str">"EleutherAI/gpt-j-6B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gpt-j-6B"</span><span class="op">,</span> <span class="str">"gpt-j"</span><span class="op">,</span> <span class="str">"gptj"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t509" href="#t509">509</a></span><span class="t">    <span class="str">"EleutherAI/gpt-neox-20b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gpt-neox-20b"</span><span class="op">,</span> <span class="str">"gpt-neox"</span><span class="op">,</span> <span class="str">"neox"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t510" href="#t510">510</a></span><span class="t">    <span class="str">"stanford-crfm/alias-gpt2-small-x21"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t511" href="#t511">511</a></span><span class="t">        <span class="str">"stanford-gpt2-small-a"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t512" href="#t512">512</a></span><span class="t">        <span class="str">"alias-gpt2-small-x21"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t513" href="#t513">513</a></span><span class="t">        <span class="str">"gpt2-mistral-small-a"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t514" href="#t514">514</a></span><span class="t">        <span class="str">"gpt2-stanford-small-a"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t515" href="#t515">515</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t516" href="#t516">516</a></span><span class="t">    <span class="str">"stanford-crfm/battlestar-gpt2-small-x49"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t517" href="#t517">517</a></span><span class="t">        <span class="str">"stanford-gpt2-small-b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t518" href="#t518">518</a></span><span class="t">        <span class="str">"battlestar-gpt2-small-x49"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t519" href="#t519">519</a></span><span class="t">        <span class="str">"gpt2-mistral-small-b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t520" href="#t520">520</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t521" href="#t521">521</a></span><span class="t">    <span class="str">"stanford-crfm/caprica-gpt2-small-x81"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t522" href="#t522">522</a></span><span class="t">        <span class="str">"stanford-gpt2-small-c"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t523" href="#t523">523</a></span><span class="t">        <span class="str">"caprica-gpt2-small-x81"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t524" href="#t524">524</a></span><span class="t">        <span class="str">"gpt2-mistral-small-c"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t525" href="#t525">525</a></span><span class="t">        <span class="str">"gpt2-stanford-small-c"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t526" href="#t526">526</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t527" href="#t527">527</a></span><span class="t">    <span class="str">"stanford-crfm/darkmatter-gpt2-small-x343"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t528" href="#t528">528</a></span><span class="t">        <span class="str">"stanford-gpt2-small-d"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t529" href="#t529">529</a></span><span class="t">        <span class="str">"darkmatter-gpt2-small-x343"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t530" href="#t530">530</a></span><span class="t">        <span class="str">"gpt2-mistral-small-d"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t520" href="#t520">520</a></span><span class="t">        <span class="str">"gpt2-mistral-small-b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t521" href="#t521">521</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t522" href="#t522">522</a></span><span class="t">    <span class="str">"stanford-crfm/caprica-gpt2-small-x81"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t523" href="#t523">523</a></span><span class="t">        <span class="str">"stanford-gpt2-small-c"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t524" href="#t524">524</a></span><span class="t">        <span class="str">"caprica-gpt2-small-x81"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t525" href="#t525">525</a></span><span class="t">        <span class="str">"gpt2-mistral-small-c"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t526" href="#t526">526</a></span><span class="t">        <span class="str">"gpt2-stanford-small-c"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t527" href="#t527">527</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t528" href="#t528">528</a></span><span class="t">    <span class="str">"stanford-crfm/darkmatter-gpt2-small-x343"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t529" href="#t529">529</a></span><span class="t">        <span class="str">"stanford-gpt2-small-d"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t530" href="#t530">530</a></span><span class="t">        <span class="str">"darkmatter-gpt2-small-x343"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t531" href="#t531">531</a></span><span class="t">        <span class="str">"gpt2-mistral-small-d"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t532" href="#t532">532</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t533" href="#t533">533</a></span><span class="t">    <span class="str">"stanford-crfm/expanse-gpt2-small-x777"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t534" href="#t534">534</a></span><span class="t">        <span class="str">"stanford-gpt2-small-e"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t535" href="#t535">535</a></span><span class="t">        <span class="str">"expanse-gpt2-small-x777"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t536" href="#t536">536</a></span><span class="t">        <span class="str">"gpt2-mistral-small-e"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t532" href="#t532">532</a></span><span class="t">        <span class="str">"gpt2-mistral-small-d"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t533" href="#t533">533</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t534" href="#t534">534</a></span><span class="t">    <span class="str">"stanford-crfm/expanse-gpt2-small-x777"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t535" href="#t535">535</a></span><span class="t">        <span class="str">"stanford-gpt2-small-e"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t536" href="#t536">536</a></span><span class="t">        <span class="str">"expanse-gpt2-small-x777"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t537" href="#t537">537</a></span><span class="t">        <span class="str">"gpt2-mistral-small-e"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t538" href="#t538">538</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t539" href="#t539">539</a></span><span class="t">    <span class="str">"stanford-crfm/arwen-gpt2-medium-x21"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t540" href="#t540">540</a></span><span class="t">        <span class="str">"stanford-gpt2-medium-a"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t541" href="#t541">541</a></span><span class="t">        <span class="str">"arwen-gpt2-medium-x21"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t542" href="#t542">542</a></span><span class="t">        <span class="str">"gpt2-medium-small-a"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t543" href="#t543">543</a></span><span class="t">        <span class="str">"gpt2-stanford-medium-a"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t544" href="#t544">544</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t545" href="#t545">545</a></span><span class="t">    <span class="str">"stanford-crfm/beren-gpt2-medium-x49"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t546" href="#t546">546</a></span><span class="t">        <span class="str">"stanford-gpt2-medium-b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t547" href="#t547">547</a></span><span class="t">        <span class="str">"beren-gpt2-medium-x49"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t548" href="#t548">548</a></span><span class="t">        <span class="str">"gpt2-medium-small-b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t549" href="#t549">549</a></span><span class="t">        <span class="str">"gpt2-stanford-medium-b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t550" href="#t550">550</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t551" href="#t551">551</a></span><span class="t">    <span class="str">"stanford-crfm/celebrimbor-gpt2-medium-x81"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t552" href="#t552">552</a></span><span class="t">        <span class="str">"stanford-gpt2-medium-c"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t553" href="#t553">553</a></span><span class="t">        <span class="str">"celebrimbor-gpt2-medium-x81"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t554" href="#t554">554</a></span><span class="t">        <span class="str">"gpt2-medium-small-c"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t538" href="#t538">538</a></span><span class="t">        <span class="str">"gpt2-mistral-small-e"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t539" href="#t539">539</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t540" href="#t540">540</a></span><span class="t">    <span class="str">"stanford-crfm/arwen-gpt2-medium-x21"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t541" href="#t541">541</a></span><span class="t">        <span class="str">"stanford-gpt2-medium-a"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t542" href="#t542">542</a></span><span class="t">        <span class="str">"arwen-gpt2-medium-x21"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t543" href="#t543">543</a></span><span class="t">        <span class="str">"gpt2-medium-small-a"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t544" href="#t544">544</a></span><span class="t">        <span class="str">"gpt2-stanford-medium-a"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t545" href="#t545">545</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t546" href="#t546">546</a></span><span class="t">    <span class="str">"stanford-crfm/beren-gpt2-medium-x49"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t547" href="#t547">547</a></span><span class="t">        <span class="str">"stanford-gpt2-medium-b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t548" href="#t548">548</a></span><span class="t">        <span class="str">"beren-gpt2-medium-x49"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t549" href="#t549">549</a></span><span class="t">        <span class="str">"gpt2-medium-small-b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t550" href="#t550">550</a></span><span class="t">        <span class="str">"gpt2-stanford-medium-b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t551" href="#t551">551</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t552" href="#t552">552</a></span><span class="t">    <span class="str">"stanford-crfm/celebrimbor-gpt2-medium-x81"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t553" href="#t553">553</a></span><span class="t">        <span class="str">"stanford-gpt2-medium-c"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t554" href="#t554">554</a></span><span class="t">        <span class="str">"celebrimbor-gpt2-medium-x81"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t555" href="#t555">555</a></span><span class="t">        <span class="str">"gpt2-medium-small-c"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t556" href="#t556">556</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t557" href="#t557">557</a></span><span class="t">    <span class="str">"stanford-crfm/durin-gpt2-medium-x343"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t558" href="#t558">558</a></span><span class="t">        <span class="str">"stanford-gpt2-medium-d"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t559" href="#t559">559</a></span><span class="t">        <span class="str">"durin-gpt2-medium-x343"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t560" href="#t560">560</a></span><span class="t">        <span class="str">"gpt2-medium-small-d"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t561" href="#t561">561</a></span><span class="t">        <span class="str">"gpt2-stanford-medium-d"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t562" href="#t562">562</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t563" href="#t563">563</a></span><span class="t">    <span class="str">"stanford-crfm/eowyn-gpt2-medium-x777"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t564" href="#t564">564</a></span><span class="t">        <span class="str">"stanford-gpt2-medium-e"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t565" href="#t565">565</a></span><span class="t">        <span class="str">"eowyn-gpt2-medium-x777"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t566" href="#t566">566</a></span><span class="t">        <span class="str">"gpt2-medium-small-e"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t567" href="#t567">567</a></span><span class="t">        <span class="str">"gpt2-stanford-medium-e"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t568" href="#t568">568</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t569" href="#t569">569</a></span><span class="t">    <span class="str">"ArthurConmy/redwood_attn_2l"</span><span class="op">:</span> <span class="op">[</span><span class="str">"redwood_attn_2l"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t570" href="#t570">570</a></span><span class="t">    <span class="str">"llama-7b-hf"</span><span class="op">:</span> <span class="op">[</span><span class="str">"llama-7b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t571" href="#t571">571</a></span><span class="t">    <span class="str">"llama-13b-hf"</span><span class="op">:</span> <span class="op">[</span><span class="str">"llama-13b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t572" href="#t572">572</a></span><span class="t">    <span class="str">"llama-30b-hf"</span><span class="op">:</span> <span class="op">[</span><span class="str">"llama-30b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t573" href="#t573">573</a></span><span class="t">    <span class="str">"llama-65b-hf"</span><span class="op">:</span> <span class="op">[</span><span class="str">"llama-65b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t574" href="#t574">574</a></span><span class="t">    <span class="str">"meta-llama/Llama-2-7b-hf"</span><span class="op">:</span> <span class="op">[</span><span class="str">"Llama-2-7b"</span><span class="op">,</span> <span class="str">"meta-llama/Llama-2-7b-hf"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t575" href="#t575">575</a></span><span class="t">    <span class="str">"meta-llama/Llama-2-7b-chat-hf"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t576" href="#t576">576</a></span><span class="t">        <span class="str">"Llama-2-7b-chat"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t577" href="#t577">577</a></span><span class="t">        <span class="str">"meta-llama/Llama-2-7b-chat-hf"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t578" href="#t578">578</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t579" href="#t579">579</a></span><span class="t">    <span class="str">"meta-llama/Llama-2-13b-hf"</span><span class="op">:</span> <span class="op">[</span><span class="str">"Llama-2-13b"</span><span class="op">,</span> <span class="str">"meta-llama/Llama-2-13b-hf"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t580" href="#t580">580</a></span><span class="t">    <span class="str">"meta-llama/Llama-2-13b-chat-hf"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t581" href="#t581">581</a></span><span class="t">        <span class="str">"Llama-2-13b-chat"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t582" href="#t582">582</a></span><span class="t">        <span class="str">"meta-llama/Llama-2-13b-chat-hf"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t583" href="#t583">583</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t584" href="#t584">584</a></span><span class="t">    <span class="str">"meta-llama/Llama-2-70b-chat-hf"</span><span class="op">:</span> <span class="op">[</span><span class="str">"Llama-2-70b-chat"</span><span class="op">,</span> <span class="str">"meta-llama-2-70b-chat-hf"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t585" href="#t585">585</a></span><span class="t">    <span class="str">"codellama/CodeLlama-7b-hf"</span><span class="op">:</span> <span class="op">[</span><span class="str">"CodeLlamallama-2-7b"</span><span class="op">,</span> <span class="str">"codellama/CodeLlama-7b-hf"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t586" href="#t586">586</a></span><span class="t">    <span class="str">"codellama/CodeLlama-7b-Python-hf"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t587" href="#t587">587</a></span><span class="t">        <span class="str">"CodeLlama-7b-python"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t588" href="#t588">588</a></span><span class="t">        <span class="str">"codellama/CodeLlama-7b-Python-hf"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t589" href="#t589">589</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t590" href="#t590">590</a></span><span class="t">    <span class="str">"codellama/CodeLlama-7b-Instruct-hf"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t591" href="#t591">591</a></span><span class="t">        <span class="str">"CodeLlama-7b-instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t592" href="#t592">592</a></span><span class="t">        <span class="str">"codellama/CodeLlama-7b-Instruct-hf"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t593" href="#t593">593</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t594" href="#t594">594</a></span><span class="t">    <span class="str">"Baidicoot/Othello-GPT-Transformer-Lens"</span><span class="op">:</span> <span class="op">[</span><span class="str">"othello-gpt"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t595" href="#t595">595</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-1M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-1M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t596" href="#t596">596</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-3M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-3M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t597" href="#t597">597</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-8M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-8M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t598" href="#t598">598</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-28M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-28M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t599" href="#t599">599</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-33M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-33M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t600" href="#t600">600</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instruct-1M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-instruct-1M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t601" href="#t601">601</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instruct-3M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-instruct-3M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t602" href="#t602">602</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instruct-8M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-instruct-8M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t603" href="#t603">603</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instruct-28M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-instruct-28M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t604" href="#t604">604</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instruct-33M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-instruct-33M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t605" href="#t605">605</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-1Layer-21M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-1L-21M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t606" href="#t606">606</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-2Layers-33M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-2L-33M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t607" href="#t607">607</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instuct-1Layer-21M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-instruct-1L-21M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t608" href="#t608">608</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instruct-2Layers-33M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-instruct-2L-33M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t609" href="#t609">609</a></span><span class="t">    <span class="str">"stabilityai/stablelm-base-alpha-3b"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t610" href="#t610">610</a></span><span class="t">        <span class="str">"stablelm-base-alpha-3b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t611" href="#t611">611</a></span><span class="t">        <span class="str">"stablelm-base-3b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t612" href="#t612">612</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t613" href="#t613">613</a></span><span class="t">    <span class="str">"stabilityai/stablelm-base-alpha-7b"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t614" href="#t614">614</a></span><span class="t">        <span class="str">"stablelm-base-alpha-7b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t615" href="#t615">615</a></span><span class="t">        <span class="str">"stablelm-base-7b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t616" href="#t616">616</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t617" href="#t617">617</a></span><span class="t">    <span class="str">"stabilityai/stablelm-tuned-alpha-3b"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t618" href="#t618">618</a></span><span class="t">        <span class="str">"stablelm-tuned-alpha-3b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t619" href="#t619">619</a></span><span class="t">        <span class="str">"stablelm-tuned-3b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t620" href="#t620">620</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t621" href="#t621">621</a></span><span class="t">    <span class="str">"stabilityai/stablelm-tuned-alpha-7b"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t622" href="#t622">622</a></span><span class="t">        <span class="str">"stablelm-tuned-alpha-7b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t623" href="#t623">623</a></span><span class="t">        <span class="str">"stablelm-tuned-7b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t624" href="#t624">624</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t625" href="#t625">625</a></span><span class="t">    <span class="str">"mistralai/Mistral-7B-v0.1"</span><span class="op">:</span> <span class="op">[</span><span class="str">"mistral-7b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t626" href="#t626">626</a></span><span class="t">    <span class="str">"mistralai/Mistral-7B-Instruct-v0.1"</span><span class="op">:</span> <span class="op">[</span><span class="str">"mistral-7b-instruct"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t627" href="#t627">627</a></span><span class="t">    <span class="str">"mistralai/Mistral-Nemo-Base-2407"</span><span class="op">:</span> <span class="op">[</span><span class="str">"mistral-nemo-base-2407"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t628" href="#t628">628</a></span><span class="t">    <span class="str">"mistralai/Mixtral-8x7B-v0.1"</span><span class="op">:</span> <span class="op">[</span><span class="str">"mixtral"</span><span class="op">,</span> <span class="str">"mixtral-8x7b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t629" href="#t629">629</a></span><span class="t">    <span class="str">"mistralai/Mixtral-8x7B-Instruct-v0.1"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t630" href="#t630">630</a></span><span class="t">        <span class="str">"mixtral-instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t631" href="#t631">631</a></span><span class="t">        <span class="str">"mixtral-8x7b-instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t632" href="#t632">632</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t633" href="#t633">633</a></span><span class="t">    <span class="str">"bigscience/bloom-560m"</span><span class="op">:</span> <span class="op">[</span><span class="str">"bloom-560m"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t634" href="#t634">634</a></span><span class="t">    <span class="str">"bigscience/bloom-1b1"</span><span class="op">:</span> <span class="op">[</span><span class="str">"bloom-1b1"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t635" href="#t635">635</a></span><span class="t">    <span class="str">"bigscience/bloom-1b7"</span><span class="op">:</span> <span class="op">[</span><span class="str">"bloom-1b7"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t636" href="#t636">636</a></span><span class="t">    <span class="str">"bigscience/bloom-3b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"bloom-3b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t637" href="#t637">637</a></span><span class="t">    <span class="str">"bigscience/bloom-7b1"</span><span class="op">:</span> <span class="op">[</span><span class="str">"bloom-7b1"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t638" href="#t638">638</a></span><span class="t">    <span class="str">"bigcode/santacoder"</span><span class="op">:</span> <span class="op">[</span><span class="str">"santacoder"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t639" href="#t639">639</a></span><span class="t">    <span class="str">"Qwen/Qwen-1_8B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen-1.8b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t640" href="#t640">640</a></span><span class="t">    <span class="str">"Qwen/Qwen-7B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen-7b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t641" href="#t641">641</a></span><span class="t">    <span class="str">"Qwen/Qwen-14B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen-14b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t642" href="#t642">642</a></span><span class="t">    <span class="str">"Qwen/Qwen-1_8B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen-1.8b-chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t643" href="#t643">643</a></span><span class="t">    <span class="str">"Qwen/Qwen-7B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen-7b-chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t644" href="#t644">644</a></span><span class="t">    <span class="str">"Qwen/Qwen-14B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen-14b-chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t645" href="#t645">645</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-0.5B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-0.5b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t646" href="#t646">646</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-0.5B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-0.5b-chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t647" href="#t647">647</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-1.8B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-1.8b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t648" href="#t648">648</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-1.8B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-1.8b-chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t649" href="#t649">649</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-4B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-4b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t650" href="#t650">650</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-4B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-4b-chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t651" href="#t651">651</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-7B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-7b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t652" href="#t652">652</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-7B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-7b-chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t653" href="#t653">653</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-14B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-14b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t654" href="#t654">654</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-14B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-14b-chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t655" href="#t655">655</a></span><span class="t">    <span class="str">"microsoft/phi-1"</span><span class="op">:</span> <span class="op">[</span><span class="str">"phi-1"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t656" href="#t656">656</a></span><span class="t">    <span class="str">"microsoft/phi-1_5"</span><span class="op">:</span> <span class="op">[</span><span class="str">"phi-1_5"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t657" href="#t657">657</a></span><span class="t">    <span class="str">"microsoft/phi-2"</span><span class="op">:</span> <span class="op">[</span><span class="str">"phi-2"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t658" href="#t658">658</a></span><span class="t">    <span class="str">"microsoft/Phi-3-mini-4k-instruct"</span><span class="op">:</span> <span class="op">[</span><span class="str">"phi-3"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t659" href="#t659">659</a></span><span class="t">    <span class="str">"google/gemma-2b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-2b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t660" href="#t660">660</a></span><span class="t">    <span class="str">"google/gemma-7b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-7b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t661" href="#t661">661</a></span><span class="t">    <span class="str">"google/gemma-2b-it"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-2b-it"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t662" href="#t662">662</a></span><span class="t">    <span class="str">"google/gemma-7b-it"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-7b-it"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t663" href="#t663">663</a></span><span class="t">    <span class="str">"google/gemma-2-2b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-2-2b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t664" href="#t664">664</a></span><span class="t">    <span class="str">"google/gemma-2-9b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-2-9b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t665" href="#t665">665</a></span><span class="t">    <span class="str">"google/gemma-2-27b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-2-27b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t666" href="#t666">666</a></span><span class="t">    <span class="str">"google/gemma-2-2b-it"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-2-2b-it"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t667" href="#t667">667</a></span><span class="t">    <span class="str">"google/gemma-2-9b-it"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-2-9b-it"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t668" href="#t668">668</a></span><span class="t">    <span class="str">"google/gemma-2-27b-it"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-2-27b-it"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t669" href="#t669">669</a></span><span class="t">    <span class="str">"01-ai/Yi-6B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"yi-6b"</span><span class="op">,</span> <span class="str">"Yi-6B"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t670" href="#t670">670</a></span><span class="t">    <span class="str">"01-ai/Yi-34B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"yi-34b"</span><span class="op">,</span> <span class="str">"Yi-34B"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t671" href="#t671">671</a></span><span class="t">    <span class="str">"01-ai/Yi-6B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"yi-6b-chat"</span><span class="op">,</span> <span class="str">"Yi-6B-Chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t672" href="#t672">672</a></span><span class="t">    <span class="str">"01-ai/Yi-34B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"yi-34b-chat"</span><span class="op">,</span> <span class="str">"Yi-34B-Chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t673" href="#t673">673</a></span><span class="t">    <span class="str">"google-t5/t5-small"</span><span class="op">:</span> <span class="op">[</span><span class="str">"t5-small"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t674" href="#t674">674</a></span><span class="t">    <span class="str">"google-t5/t5-base"</span><span class="op">:</span> <span class="op">[</span><span class="str">"t5-base"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t675" href="#t675">675</a></span><span class="t">    <span class="str">"google-t5/t5-large"</span><span class="op">:</span> <span class="op">[</span><span class="str">"t5-large"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t676" href="#t676">676</a></span><span class="t">    <span class="str">"ai-forever/mGPT"</span><span class="op">:</span> <span class="op">[</span><span class="str">"mGPT"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t677" href="#t677">677</a></span><span class="t"><span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t678" href="#t678">678</a></span><span class="t"><span class="str">"""Model aliases for models on HuggingFace."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t679" href="#t679">679</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t680" href="#t680">680</a></span><span class="t"><span class="nam">NON_HF_HOSTED_MODEL_NAMES</span> <span class="op">=</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t681" href="#t681">681</a></span><span class="t">    <span class="str">"llama-7b-hf"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t682" href="#t682">682</a></span><span class="t">    <span class="str">"llama-13b-hf"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t683" href="#t683">683</a></span><span class="t">    <span class="str">"llama-30b-hf"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t684" href="#t684">684</a></span><span class="t">    <span class="str">"llama-65b-hf"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t685" href="#t685">685</a></span><span class="t"><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t686" href="#t686">686</a></span><span class="t"><span class="str">"""Official model names for models not hosted on HuggingFace."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t687" href="#t687">687</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t688" href="#t688">688</a></span><span class="t"><span class="com"># Sets a default model alias, by convention the first one in the model alias table, else the official name if it has no aliases</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t689" href="#t689">689</a></span><span class="t"><span class="nam">DEFAULT_MODEL_ALIASES</span> <span class="op">=</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t690" href="#t690">690</a></span><span class="t">    <span class="nam">MODEL_ALIASES</span><span class="op">[</span><span class="nam">name</span><span class="op">]</span><span class="op">[</span><span class="num">0</span><span class="op">]</span> <span class="key">if</span> <span class="nam">name</span> <span class="key">in</span> <span class="nam">MODEL_ALIASES</span> <span class="key">else</span> <span class="nam">name</span> <span class="key">for</span> <span class="nam">name</span> <span class="key">in</span> <span class="nam">OFFICIAL_MODEL_NAMES</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t691" href="#t691">691</a></span><span class="t"><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t692" href="#t692">692</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t693" href="#t693">693</a></span><span class="t"><span class="nam">NEED_REMOTE_CODE_MODELS</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t694" href="#t694">694</a></span><span class="t">    <span class="str">"bigcode/santacoder"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t695" href="#t695">695</a></span><span class="t">    <span class="str">"Qwen/Qwen-"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t696" href="#t696">696</a></span><span class="t">    <span class="str">"microsoft/phi-2"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t697" href="#t697">697</a></span><span class="t">    <span class="str">"microsoft/Phi-3-mini-4k-instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t698" href="#t698">698</a></span><span class="t"><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t699" href="#t699">699</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t556" href="#t556">556</a></span><span class="t">        <span class="str">"gpt2-medium-small-c"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t557" href="#t557">557</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t558" href="#t558">558</a></span><span class="t">    <span class="str">"stanford-crfm/durin-gpt2-medium-x343"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t559" href="#t559">559</a></span><span class="t">        <span class="str">"stanford-gpt2-medium-d"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t560" href="#t560">560</a></span><span class="t">        <span class="str">"durin-gpt2-medium-x343"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t561" href="#t561">561</a></span><span class="t">        <span class="str">"gpt2-medium-small-d"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t562" href="#t562">562</a></span><span class="t">        <span class="str">"gpt2-stanford-medium-d"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t563" href="#t563">563</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t564" href="#t564">564</a></span><span class="t">    <span class="str">"stanford-crfm/eowyn-gpt2-medium-x777"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t565" href="#t565">565</a></span><span class="t">        <span class="str">"stanford-gpt2-medium-e"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t566" href="#t566">566</a></span><span class="t">        <span class="str">"eowyn-gpt2-medium-x777"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t567" href="#t567">567</a></span><span class="t">        <span class="str">"gpt2-medium-small-e"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t568" href="#t568">568</a></span><span class="t">        <span class="str">"gpt2-stanford-medium-e"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t569" href="#t569">569</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t570" href="#t570">570</a></span><span class="t">    <span class="str">"ArthurConmy/redwood_attn_2l"</span><span class="op">:</span> <span class="op">[</span><span class="str">"redwood_attn_2l"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t571" href="#t571">571</a></span><span class="t">    <span class="str">"llama-7b-hf"</span><span class="op">:</span> <span class="op">[</span><span class="str">"llama-7b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t572" href="#t572">572</a></span><span class="t">    <span class="str">"llama-13b-hf"</span><span class="op">:</span> <span class="op">[</span><span class="str">"llama-13b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t573" href="#t573">573</a></span><span class="t">    <span class="str">"llama-30b-hf"</span><span class="op">:</span> <span class="op">[</span><span class="str">"llama-30b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t574" href="#t574">574</a></span><span class="t">    <span class="str">"llama-65b-hf"</span><span class="op">:</span> <span class="op">[</span><span class="str">"llama-65b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t575" href="#t575">575</a></span><span class="t">    <span class="str">"meta-llama/Llama-2-7b-hf"</span><span class="op">:</span> <span class="op">[</span><span class="str">"Llama-2-7b"</span><span class="op">,</span> <span class="str">"meta-llama/Llama-2-7b-hf"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t576" href="#t576">576</a></span><span class="t">    <span class="str">"meta-llama/Llama-2-7b-chat-hf"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t577" href="#t577">577</a></span><span class="t">        <span class="str">"Llama-2-7b-chat"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t578" href="#t578">578</a></span><span class="t">        <span class="str">"meta-llama/Llama-2-7b-chat-hf"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t579" href="#t579">579</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t580" href="#t580">580</a></span><span class="t">    <span class="str">"meta-llama/Llama-2-13b-hf"</span><span class="op">:</span> <span class="op">[</span><span class="str">"Llama-2-13b"</span><span class="op">,</span> <span class="str">"meta-llama/Llama-2-13b-hf"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t581" href="#t581">581</a></span><span class="t">    <span class="str">"meta-llama/Llama-2-13b-chat-hf"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t582" href="#t582">582</a></span><span class="t">        <span class="str">"Llama-2-13b-chat"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t583" href="#t583">583</a></span><span class="t">        <span class="str">"meta-llama/Llama-2-13b-chat-hf"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t584" href="#t584">584</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t585" href="#t585">585</a></span><span class="t">    <span class="str">"meta-llama/Llama-2-70b-chat-hf"</span><span class="op">:</span> <span class="op">[</span><span class="str">"Llama-2-70b-chat"</span><span class="op">,</span> <span class="str">"meta-llama-2-70b-chat-hf"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t586" href="#t586">586</a></span><span class="t">    <span class="str">"codellama/CodeLlama-7b-hf"</span><span class="op">:</span> <span class="op">[</span><span class="str">"CodeLlamallama-2-7b"</span><span class="op">,</span> <span class="str">"codellama/CodeLlama-7b-hf"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t587" href="#t587">587</a></span><span class="t">    <span class="str">"codellama/CodeLlama-7b-Python-hf"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t588" href="#t588">588</a></span><span class="t">        <span class="str">"CodeLlama-7b-python"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t589" href="#t589">589</a></span><span class="t">        <span class="str">"codellama/CodeLlama-7b-Python-hf"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t590" href="#t590">590</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t591" href="#t591">591</a></span><span class="t">    <span class="str">"codellama/CodeLlama-7b-Instruct-hf"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t592" href="#t592">592</a></span><span class="t">        <span class="str">"CodeLlama-7b-instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t593" href="#t593">593</a></span><span class="t">        <span class="str">"codellama/CodeLlama-7b-Instruct-hf"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t594" href="#t594">594</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t595" href="#t595">595</a></span><span class="t">    <span class="str">"Baidicoot/Othello-GPT-Transformer-Lens"</span><span class="op">:</span> <span class="op">[</span><span class="str">"othello-gpt"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t596" href="#t596">596</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-1M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-1M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t597" href="#t597">597</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-3M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-3M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t598" href="#t598">598</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-8M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-8M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t599" href="#t599">599</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-28M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-28M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t600" href="#t600">600</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-33M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-33M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t601" href="#t601">601</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instruct-1M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-instruct-1M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t602" href="#t602">602</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instruct-3M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-instruct-3M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t603" href="#t603">603</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instruct-8M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-instruct-8M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t604" href="#t604">604</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instruct-28M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-instruct-28M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t605" href="#t605">605</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instruct-33M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-instruct-33M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t606" href="#t606">606</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-1Layer-21M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-1L-21M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t607" href="#t607">607</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-2Layers-33M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-2L-33M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t608" href="#t608">608</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instuct-1Layer-21M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-instruct-1L-21M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t609" href="#t609">609</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instruct-2Layers-33M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-instruct-2L-33M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t610" href="#t610">610</a></span><span class="t">    <span class="str">"stabilityai/stablelm-base-alpha-3b"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t611" href="#t611">611</a></span><span class="t">        <span class="str">"stablelm-base-alpha-3b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t612" href="#t612">612</a></span><span class="t">        <span class="str">"stablelm-base-3b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t613" href="#t613">613</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t614" href="#t614">614</a></span><span class="t">    <span class="str">"stabilityai/stablelm-base-alpha-7b"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t615" href="#t615">615</a></span><span class="t">        <span class="str">"stablelm-base-alpha-7b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t616" href="#t616">616</a></span><span class="t">        <span class="str">"stablelm-base-7b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t617" href="#t617">617</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t618" href="#t618">618</a></span><span class="t">    <span class="str">"stabilityai/stablelm-tuned-alpha-3b"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t619" href="#t619">619</a></span><span class="t">        <span class="str">"stablelm-tuned-alpha-3b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t620" href="#t620">620</a></span><span class="t">        <span class="str">"stablelm-tuned-3b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t621" href="#t621">621</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t622" href="#t622">622</a></span><span class="t">    <span class="str">"stabilityai/stablelm-tuned-alpha-7b"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t623" href="#t623">623</a></span><span class="t">        <span class="str">"stablelm-tuned-alpha-7b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t624" href="#t624">624</a></span><span class="t">        <span class="str">"stablelm-tuned-7b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t625" href="#t625">625</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t626" href="#t626">626</a></span><span class="t">    <span class="str">"mistralai/Mistral-7B-v0.1"</span><span class="op">:</span> <span class="op">[</span><span class="str">"mistral-7b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t627" href="#t627">627</a></span><span class="t">    <span class="str">"mistralai/Mistral-7B-Instruct-v0.1"</span><span class="op">:</span> <span class="op">[</span><span class="str">"mistral-7b-instruct"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t628" href="#t628">628</a></span><span class="t">    <span class="str">"mistralai/Mistral-Nemo-Base-2407"</span><span class="op">:</span> <span class="op">[</span><span class="str">"mistral-nemo-base-2407"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t629" href="#t629">629</a></span><span class="t">    <span class="str">"mistralai/Mixtral-8x7B-v0.1"</span><span class="op">:</span> <span class="op">[</span><span class="str">"mixtral"</span><span class="op">,</span> <span class="str">"mixtral-8x7b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t630" href="#t630">630</a></span><span class="t">    <span class="str">"mistralai/Mixtral-8x7B-Instruct-v0.1"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t631" href="#t631">631</a></span><span class="t">        <span class="str">"mixtral-instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t632" href="#t632">632</a></span><span class="t">        <span class="str">"mixtral-8x7b-instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t633" href="#t633">633</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t634" href="#t634">634</a></span><span class="t">    <span class="str">"bigscience/bloom-560m"</span><span class="op">:</span> <span class="op">[</span><span class="str">"bloom-560m"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t635" href="#t635">635</a></span><span class="t">    <span class="str">"bigscience/bloom-1b1"</span><span class="op">:</span> <span class="op">[</span><span class="str">"bloom-1b1"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t636" href="#t636">636</a></span><span class="t">    <span class="str">"bigscience/bloom-1b7"</span><span class="op">:</span> <span class="op">[</span><span class="str">"bloom-1b7"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t637" href="#t637">637</a></span><span class="t">    <span class="str">"bigscience/bloom-3b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"bloom-3b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t638" href="#t638">638</a></span><span class="t">    <span class="str">"bigscience/bloom-7b1"</span><span class="op">:</span> <span class="op">[</span><span class="str">"bloom-7b1"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t639" href="#t639">639</a></span><span class="t">    <span class="str">"bigcode/santacoder"</span><span class="op">:</span> <span class="op">[</span><span class="str">"santacoder"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t640" href="#t640">640</a></span><span class="t">    <span class="str">"Qwen/Qwen-1_8B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen-1.8b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t641" href="#t641">641</a></span><span class="t">    <span class="str">"Qwen/Qwen-7B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen-7b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t642" href="#t642">642</a></span><span class="t">    <span class="str">"Qwen/Qwen-14B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen-14b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t643" href="#t643">643</a></span><span class="t">    <span class="str">"Qwen/Qwen-1_8B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen-1.8b-chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t644" href="#t644">644</a></span><span class="t">    <span class="str">"Qwen/Qwen-7B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen-7b-chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t645" href="#t645">645</a></span><span class="t">    <span class="str">"Qwen/Qwen-14B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen-14b-chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t646" href="#t646">646</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-0.5B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-0.5b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t647" href="#t647">647</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-0.5B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-0.5b-chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t648" href="#t648">648</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-1.8B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-1.8b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t649" href="#t649">649</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-1.8B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-1.8b-chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t650" href="#t650">650</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-4B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-4b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t651" href="#t651">651</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-4B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-4b-chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t652" href="#t652">652</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-7B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-7b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t653" href="#t653">653</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-7B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-7b-chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t654" href="#t654">654</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-14B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-14b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t655" href="#t655">655</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-14B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-14b-chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t656" href="#t656">656</a></span><span class="t">    <span class="str">"microsoft/phi-1"</span><span class="op">:</span> <span class="op">[</span><span class="str">"phi-1"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t657" href="#t657">657</a></span><span class="t">    <span class="str">"microsoft/phi-1_5"</span><span class="op">:</span> <span class="op">[</span><span class="str">"phi-1_5"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t658" href="#t658">658</a></span><span class="t">    <span class="str">"microsoft/phi-2"</span><span class="op">:</span> <span class="op">[</span><span class="str">"phi-2"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t659" href="#t659">659</a></span><span class="t">    <span class="str">"microsoft/Phi-3-mini-4k-instruct"</span><span class="op">:</span> <span class="op">[</span><span class="str">"phi-3"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t660" href="#t660">660</a></span><span class="t">    <span class="str">"google/gemma-2b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-2b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t661" href="#t661">661</a></span><span class="t">    <span class="str">"google/gemma-7b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-7b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t662" href="#t662">662</a></span><span class="t">    <span class="str">"google/gemma-2b-it"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-2b-it"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t663" href="#t663">663</a></span><span class="t">    <span class="str">"google/gemma-7b-it"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-7b-it"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t664" href="#t664">664</a></span><span class="t">    <span class="str">"google/gemma-2-2b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-2-2b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t665" href="#t665">665</a></span><span class="t">    <span class="str">"google/gemma-2-9b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-2-9b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t666" href="#t666">666</a></span><span class="t">    <span class="str">"google/gemma-2-27b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-2-27b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t667" href="#t667">667</a></span><span class="t">    <span class="str">"google/gemma-2-2b-it"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-2-2b-it"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t668" href="#t668">668</a></span><span class="t">    <span class="str">"google/gemma-2-9b-it"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-2-9b-it"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t669" href="#t669">669</a></span><span class="t">    <span class="str">"google/gemma-2-27b-it"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-2-27b-it"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t670" href="#t670">670</a></span><span class="t">    <span class="str">"01-ai/Yi-6B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"yi-6b"</span><span class="op">,</span> <span class="str">"Yi-6B"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t671" href="#t671">671</a></span><span class="t">    <span class="str">"01-ai/Yi-34B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"yi-34b"</span><span class="op">,</span> <span class="str">"Yi-34B"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t672" href="#t672">672</a></span><span class="t">    <span class="str">"01-ai/Yi-6B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"yi-6b-chat"</span><span class="op">,</span> <span class="str">"Yi-6B-Chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t673" href="#t673">673</a></span><span class="t">    <span class="str">"01-ai/Yi-34B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"yi-34b-chat"</span><span class="op">,</span> <span class="str">"Yi-34B-Chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t674" href="#t674">674</a></span><span class="t">    <span class="str">"google-t5/t5-small"</span><span class="op">:</span> <span class="op">[</span><span class="str">"t5-small"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t675" href="#t675">675</a></span><span class="t">    <span class="str">"google-t5/t5-base"</span><span class="op">:</span> <span class="op">[</span><span class="str">"t5-base"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t676" href="#t676">676</a></span><span class="t">    <span class="str">"google-t5/t5-large"</span><span class="op">:</span> <span class="op">[</span><span class="str">"t5-large"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t677" href="#t677">677</a></span><span class="t">    <span class="str">"ai-forever/mGPT"</span><span class="op">:</span> <span class="op">[</span><span class="str">"mGPT"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t678" href="#t678">678</a></span><span class="t"><span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t679" href="#t679">679</a></span><span class="t"><span class="str">"""Model aliases for models on HuggingFace."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t680" href="#t680">680</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t681" href="#t681">681</a></span><span class="t"><span class="nam">NON_HF_HOSTED_MODEL_NAMES</span> <span class="op">=</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t682" href="#t682">682</a></span><span class="t">    <span class="str">"llama-7b-hf"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t683" href="#t683">683</a></span><span class="t">    <span class="str">"llama-13b-hf"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t684" href="#t684">684</a></span><span class="t">    <span class="str">"llama-30b-hf"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t685" href="#t685">685</a></span><span class="t">    <span class="str">"llama-65b-hf"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t686" href="#t686">686</a></span><span class="t"><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t687" href="#t687">687</a></span><span class="t"><span class="str">"""Official model names for models not hosted on HuggingFace."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t688" href="#t688">688</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t689" href="#t689">689</a></span><span class="t"><span class="com"># Sets a default model alias, by convention the first one in the model alias table, else the official name if it has no aliases</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t690" href="#t690">690</a></span><span class="t"><span class="nam">DEFAULT_MODEL_ALIASES</span> <span class="op">=</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t691" href="#t691">691</a></span><span class="t">    <span class="nam">MODEL_ALIASES</span><span class="op">[</span><span class="nam">name</span><span class="op">]</span><span class="op">[</span><span class="num">0</span><span class="op">]</span> <span class="key">if</span> <span class="nam">name</span> <span class="key">in</span> <span class="nam">MODEL_ALIASES</span> <span class="key">else</span> <span class="nam">name</span> <span class="key">for</span> <span class="nam">name</span> <span class="key">in</span> <span class="nam">OFFICIAL_MODEL_NAMES</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t692" href="#t692">692</a></span><span class="t"><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t693" href="#t693">693</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t694" href="#t694">694</a></span><span class="t"><span class="nam">NEED_REMOTE_CODE_MODELS</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t695" href="#t695">695</a></span><span class="t">    <span class="str">"bigcode/santacoder"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t696" href="#t696">696</a></span><span class="t">    <span class="str">"Qwen/Qwen-"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t697" href="#t697">697</a></span><span class="t">    <span class="str">"microsoft/phi-2"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t698" href="#t698">698</a></span><span class="t">    <span class="str">"microsoft/Phi-3-mini-4k-instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t699" href="#t699">699</a></span><span class="t"><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t700" href="#t700">700</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t701" href="#t701">701</a></span><span class="t"><span class="key">def</span> <span class="nam">make_model_alias_map</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t702" href="#t702">702</a></span><span class="t">    <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t703" href="#t703">703</a></span><span class="t"><span class="str">    Converts OFFICIAL_MODEL_NAMES (the list of actual model names on</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t704" href="#t704">704</a></span><span class="t"><span class="str">    HuggingFace) and MODEL_ALIASES (a dictionary mapping official model names to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t705" href="#t705">705</a></span><span class="t"><span class="str">    aliases) into a dictionary mapping all aliases to the official model name.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t706" href="#t706">706</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t707" href="#t707">707</a></span><span class="t">    <span class="nam">model_alias_map</span> <span class="op">=</span> <span class="op">{</span><span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t708" href="#t708">708</a></span><span class="t">    <span class="key">for</span> <span class="nam">official_model_name</span> <span class="key">in</span> <span class="nam">OFFICIAL_MODEL_NAMES</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t709" href="#t709">709</a></span><span class="t">        <span class="nam">aliases</span> <span class="op">=</span> <span class="nam">MODEL_ALIASES</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">,</span> <span class="op">[</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t710" href="#t710">710</a></span><span class="t">        <span class="key">for</span> <span class="nam">alias</span> <span class="key">in</span> <span class="nam">aliases</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t711" href="#t711">711</a></span><span class="t">            <span class="nam">model_alias_map</span><span class="op">[</span><span class="nam">alias</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span><span class="op">]</span> <span class="op">=</span> <span class="nam">official_model_name</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t712" href="#t712">712</a></span><span class="t">        <span class="nam">model_alias_map</span><span class="op">[</span><span class="nam">official_model_name</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span><span class="op">]</span> <span class="op">=</span> <span class="nam">official_model_name</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t713" href="#t713">713</a></span><span class="t">    <span class="key">return</span> <span class="nam">model_alias_map</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t714" href="#t714">714</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t701" href="#t701">701</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t702" href="#t702">702</a></span><span class="t"><span class="key">def</span> <span class="nam">make_model_alias_map</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t703" href="#t703">703</a></span><span class="t">    <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t704" href="#t704">704</a></span><span class="t"><span class="str">    Converts OFFICIAL_MODEL_NAMES (the list of actual model names on</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t705" href="#t705">705</a></span><span class="t"><span class="str">    HuggingFace) and MODEL_ALIASES (a dictionary mapping official model names to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t706" href="#t706">706</a></span><span class="t"><span class="str">    aliases) into a dictionary mapping all aliases to the official model name.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t707" href="#t707">707</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t708" href="#t708">708</a></span><span class="t">    <span class="nam">model_alias_map</span> <span class="op">=</span> <span class="op">{</span><span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t709" href="#t709">709</a></span><span class="t">    <span class="key">for</span> <span class="nam">official_model_name</span> <span class="key">in</span> <span class="nam">OFFICIAL_MODEL_NAMES</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t710" href="#t710">710</a></span><span class="t">        <span class="nam">aliases</span> <span class="op">=</span> <span class="nam">MODEL_ALIASES</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">,</span> <span class="op">[</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t711" href="#t711">711</a></span><span class="t">        <span class="key">for</span> <span class="nam">alias</span> <span class="key">in</span> <span class="nam">aliases</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t712" href="#t712">712</a></span><span class="t">            <span class="nam">model_alias_map</span><span class="op">[</span><span class="nam">alias</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span><span class="op">]</span> <span class="op">=</span> <span class="nam">official_model_name</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t713" href="#t713">713</a></span><span class="t">        <span class="nam">model_alias_map</span><span class="op">[</span><span class="nam">official_model_name</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span><span class="op">]</span> <span class="op">=</span> <span class="nam">official_model_name</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t714" href="#t714">714</a></span><span class="t">    <span class="key">return</span> <span class="nam">model_alias_map</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t715" href="#t715">715</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t716" href="#t716">716</a></span><span class="t"><span class="key">def</span> <span class="nam">get_official_model_name</span><span class="op">(</span><span class="nam">model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t717" href="#t717">717</a></span><span class="t">    <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t718" href="#t718">718</a></span><span class="t"><span class="str">    Returns the official model name for a given model name (or alias).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t719" href="#t719">719</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t720" href="#t720">720</a></span><span class="t">    <span class="nam">model_alias_map</span> <span class="op">=</span> <span class="nam">make_model_alias_map</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t721" href="#t721">721</a></span><span class="t">    <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">model_alias_map</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="nam">model_name</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span><span class="op">,</span> <span class="key">None</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t722" href="#t722">722</a></span><span class="t">    <span class="key">if</span> <span class="nam">official_model_name</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">722&#x202F;&#x219B;&#x202F;723</span><span class="annotate long">line 722 didn't jump to line 723, because the condition on line 722 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t723" href="#t723">723</a></span><span class="t">        <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t724" href="#t724">724</a></span><span class="t">            <span class="str">f"{model_name} not found. Valid official model names (excl aliases): {OFFICIAL_MODEL_NAMES}"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t725" href="#t725">725</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t726" href="#t726">726</a></span><span class="t">    <span class="key">return</span> <span class="nam">official_model_name</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t727" href="#t727">727</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t716" href="#t716">716</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t717" href="#t717">717</a></span><span class="t"><span class="key">def</span> <span class="nam">get_official_model_name</span><span class="op">(</span><span class="nam">model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t718" href="#t718">718</a></span><span class="t">    <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t719" href="#t719">719</a></span><span class="t"><span class="str">    Returns the official model name for a given model name (or alias).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t720" href="#t720">720</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t721" href="#t721">721</a></span><span class="t">    <span class="nam">model_alias_map</span> <span class="op">=</span> <span class="nam">make_model_alias_map</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t722" href="#t722">722</a></span><span class="t">    <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">model_alias_map</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="nam">model_name</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span><span class="op">,</span> <span class="key">None</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t723" href="#t723">723</a></span><span class="t">    <span class="key">if</span> <span class="nam">official_model_name</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">723&#x202F;&#x219B;&#x202F;724</span><span class="annotate long">line 723 didn't jump to line 724, because the condition on line 723 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t724" href="#t724">724</a></span><span class="t">        <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t725" href="#t725">725</a></span><span class="t">            <span class="str">f"{model_name} not found. Valid official model names (excl aliases): {OFFICIAL_MODEL_NAMES}"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t726" href="#t726">726</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t727" href="#t727">727</a></span><span class="t">    <span class="key">return</span> <span class="nam">official_model_name</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t728" href="#t728">728</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t729" href="#t729">729</a></span><span class="t"><span class="key">def</span> <span class="nam">convert_hf_model_config</span><span class="op">(</span><span class="nam">model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t730" href="#t730">730</a></span><span class="t">    <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t731" href="#t731">731</a></span><span class="t"><span class="str">    Returns the model config for a HuggingFace model, converted to a dictionary</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t732" href="#t732">732</a></span><span class="t"><span class="str">    in the HookedTransformerConfig format.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t733" href="#t733">733</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t734" href="#t734">734</a></span><span class="t"><span class="str">    Takes the official_model_name as an input.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t735" href="#t735">735</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t736" href="#t736">736</a></span><span class="t">    <span class="com"># In case the user passed in an alias</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t737" href="#t737">737</a></span><span class="t">    <span class="key">if</span> <span class="op">(</span><span class="nam">Path</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span> <span class="op">/</span> <span class="str">"config.json"</span><span class="op">)</span><span class="op">.</span><span class="nam">exists</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">737&#x202F;&#x219B;&#x202F;738</span><span class="annotate long">line 737 didn't jump to line 738, because the condition on line 737 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t738" href="#t738">738</a></span><span class="t">        <span class="nam">logging</span><span class="op">.</span><span class="nam">info</span><span class="op">(</span><span class="str">"Loading model config from local directory"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t739" href="#t739">739</a></span><span class="t">        <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">model_name</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t740" href="#t740">740</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t741" href="#t741">741</a></span><span class="t">        <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">get_official_model_name</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t742" href="#t742">742</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t743" href="#t743">743</a></span><span class="t">    <span class="com"># Load HuggingFace model config</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t744" href="#t744">744</a></span><span class="t">    <span class="key">if</span> <span class="str">"llama"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">744&#x202F;&#x219B;&#x202F;745</span><span class="annotate long">line 744 didn't jump to line 745, because the condition on line 744 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t745" href="#t745">745</a></span><span class="t">        <span class="nam">architecture</span> <span class="op">=</span> <span class="str">"LlamaForCausalLM"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t746" href="#t746">746</a></span><span class="t">    <span class="key">elif</span> <span class="str">"gemma-2"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">746&#x202F;&#x219B;&#x202F;747</span><span class="annotate long">line 746 didn't jump to line 747, because the condition on line 746 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t747" href="#t747">747</a></span><span class="t">        <span class="nam">architecture</span> <span class="op">=</span> <span class="str">"Gemma2ForCausalLM"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t748" href="#t748">748</a></span><span class="t">    <span class="key">elif</span> <span class="str">"gemma"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">748&#x202F;&#x219B;&#x202F;749</span><span class="annotate long">line 748 didn't jump to line 749, because the condition on line 748 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t749" href="#t749">749</a></span><span class="t">        <span class="nam">architecture</span> <span class="op">=</span> <span class="str">"GemmaForCausalLM"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t750" href="#t750">750</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t751" href="#t751">751</a></span><span class="t">        <span class="nam">huggingface_token</span> <span class="op">=</span> <span class="nam">os</span><span class="op">.</span><span class="nam">environ</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"HF_TOKEN"</span><span class="op">,</span> <span class="key">None</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t752" href="#t752">752</a></span><span class="t">        <span class="nam">hf_config</span> <span class="op">=</span> <span class="nam">AutoConfig</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t753" href="#t753">753</a></span><span class="t">            <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t754" href="#t754">754</a></span><span class="t">            <span class="nam">token</span><span class="op">=</span><span class="nam">huggingface_token</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t755" href="#t755">755</a></span><span class="t">            <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t756" href="#t756">756</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t757" href="#t757">757</a></span><span class="t">        <span class="nam">architecture</span> <span class="op">=</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">architectures</span><span class="op">[</span><span class="num">0</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t758" href="#t758">758</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t759" href="#t759">759</a></span><span class="t">    <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span>&nbsp;</span><span class="r"><span class="annotate short">759&#x202F;&#x219B;&#x202F;762</span><span class="annotate long">line 759 didn't jump to line 762</span></span></p>
-    <p class="pln"><span class="n"><a id="t760" href="#t760">760</a></span><span class="t">        <span class="op">(</span><span class="str">"llama-7b"</span><span class="op">,</span> <span class="str">"meta-llama/Llama-2-7b"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t761" href="#t761">761</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>  <span class="com"># same architecture for LLaMA and Llama-2</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t762" href="#t762">762</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t763" href="#t763">763</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t764" href="#t764">764</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">4096</span> <span class="op">//</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t765" href="#t765">765</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t766" href="#t766">766</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">11008</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t767" href="#t767">767</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t768" href="#t768">768</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span> <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"llama-7b"</span><span class="op">)</span> <span class="key">else</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t769" href="#t769">769</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-6</span> <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"llama-7b"</span><span class="op">)</span> <span class="key">else</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t770" href="#t770">770</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">32000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t771" href="#t771">771</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t772" href="#t772">772</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t773" href="#t773">773</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t774" href="#t774">774</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t775" href="#t775">775</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">4096</span> <span class="op">//</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t776" href="#t776">776</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t777" href="#t777">777</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t778" href="#t778">778</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t779" href="#t779">779</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"codellama"</span><span class="op">)</span><span class="op">:</span>  <span class="com"># same architecture CodeLlama and Llama-2</span>&nbsp;</span><span class="r"><span class="annotate short">779&#x202F;&#x219B;&#x202F;780</span><span class="annotate long">line 779 didn't jump to line 780</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t780" href="#t780">780</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t781" href="#t781">781</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t782" href="#t782">782</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">4096</span> <span class="op">//</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t783" href="#t783">783</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t784" href="#t784">784</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">11008</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t785" href="#t785">785</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t786" href="#t786">786</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t787" href="#t787">787</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t788" href="#t788">788</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">32016</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t789" href="#t789">789</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t790" href="#t790">790</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t791" href="#t791">791</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t792" href="#t792">792</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">4096</span> <span class="op">//</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t793" href="#t793">793</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t794" href="#t794">794</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t795" href="#t795">795</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">1000000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t796" href="#t796">796</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t797" href="#t797">797</a></span><span class="t">        <span class="key">if</span> <span class="str">"python"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t798" href="#t798">798</a></span><span class="t">            <span class="com"># The vocab size of python version of CodeLlama-7b is 32000</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t799" href="#t799">799</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"d_vocab"</span><span class="op">]</span> <span class="op">=</span> <span class="num">32000</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t800" href="#t800">800</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span>&nbsp;</span><span class="r"><span class="annotate short">800&#x202F;&#x219B;&#x202F;803</span><span class="annotate long">line 800 didn't jump to line 803</span></span></p>
-    <p class="pln"><span class="n"><a id="t801" href="#t801">801</a></span><span class="t">        <span class="op">(</span><span class="str">"llama-13b"</span><span class="op">,</span> <span class="str">"meta-llama/Llama-2-13b"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t802" href="#t802">802</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>  <span class="com"># same architecture for LLaMA and Llama-2</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t803" href="#t803">803</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t804" href="#t804">804</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">5120</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t805" href="#t805">805</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">5120</span> <span class="op">//</span> <span class="num">40</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t806" href="#t806">806</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">40</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t807" href="#t807">807</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">13824</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t808" href="#t808">808</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">40</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t809" href="#t809">809</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span> <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"llama-13b"</span><span class="op">)</span> <span class="key">else</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t810" href="#t810">810</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-6</span> <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"llama-13b"</span><span class="op">)</span> <span class="key">else</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t811" href="#t811">811</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">32000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t812" href="#t812">812</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t813" href="#t813">813</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t814" href="#t814">814</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t815" href="#t815">815</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t816" href="#t816">816</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">5120</span> <span class="op">//</span> <span class="num">40</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t817" href="#t817">817</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t818" href="#t818">818</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t819" href="#t819">819</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t820" href="#t820">820</a></span><span class="t">    <span class="key">elif</span> <span class="str">"llama-30b"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">820&#x202F;&#x219B;&#x202F;821</span><span class="annotate long">line 820 didn't jump to line 821</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t821" href="#t821">821</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t822" href="#t822">822</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">6656</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t823" href="#t823">823</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">6656</span> <span class="op">//</span> <span class="num">52</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t824" href="#t824">824</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">52</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t825" href="#t825">825</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">17920</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t826" href="#t826">826</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">60</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t827" href="#t827">827</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t828" href="#t828">828</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-6</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t829" href="#t829">829</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">32000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t830" href="#t830">830</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t831" href="#t831">831</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t832" href="#t832">832</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t833" href="#t833">833</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t834" href="#t834">834</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">6656</span> <span class="op">//</span> <span class="num">52</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t835" href="#t835">835</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t836" href="#t836">836</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t837" href="#t837">837</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t838" href="#t838">838</a></span><span class="t">    <span class="key">elif</span> <span class="str">"llama-65b"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">838&#x202F;&#x219B;&#x202F;839</span><span class="annotate long">line 838 didn't jump to line 839</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t839" href="#t839">839</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t840" href="#t840">840</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t841" href="#t841">841</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">8192</span> <span class="op">//</span> <span class="num">64</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t842" href="#t842">842</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">64</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t843" href="#t843">843</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">22016</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t844" href="#t844">844</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">80</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t845" href="#t845">845</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t846" href="#t846">846</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-6</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t847" href="#t847">847</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">32000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t848" href="#t848">848</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t849" href="#t849">849</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t850" href="#t850">850</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t851" href="#t851">851</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">8192</span> <span class="op">//</span> <span class="num">64</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t852" href="#t852">852</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t853" href="#t853">853</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t854" href="#t854">854</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t855" href="#t855">855</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t856" href="#t856">856</a></span><span class="t">    <span class="key">elif</span> <span class="str">"Llama-2-70b"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">856&#x202F;&#x219B;&#x202F;857</span><span class="annotate long">line 856 didn't jump to line 857</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t857" href="#t857">857</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t858" href="#t858">858</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t859" href="#t859">859</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t860" href="#t860">860</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">64</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t861" href="#t861">861</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">28672</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t862" href="#t862">862</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">80</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t863" href="#t863">863</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t864" href="#t864">864</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t865" href="#t865">865</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">32000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t866" href="#t866">866</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t867" href="#t867">867</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t868" href="#t868">868</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t869" href="#t869">869</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t870" href="#t870">870</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t871" href="#t871">871</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t872" href="#t872">872</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t873" href="#t873">873</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t874" href="#t874">874</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t875" href="#t875">875</a></span><span class="t">    <span class="key">elif</span> <span class="str">"Meta-Llama-3-8B"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">875&#x202F;&#x219B;&#x202F;876</span><span class="annotate long">line 875 didn't jump to line 876</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t876" href="#t876">876</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t877" href="#t877">877</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t878" href="#t878">878</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t879" href="#t879">879</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t880" href="#t880">880</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">14336</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t881" href="#t881">881</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t882" href="#t882">882</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t883" href="#t883">883</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t884" href="#t884">884</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">128256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t885" href="#t885">885</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t886" href="#t886">886</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t887" href="#t887">887</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t888" href="#t888">888</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t889" href="#t889">889</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t890" href="#t890">890</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t891" href="#t891">891</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t892" href="#t892">892</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t893" href="#t893">893</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">500000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t894" href="#t894">894</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t895" href="#t895">895</a></span><span class="t">    <span class="key">elif</span> <span class="str">"Meta-Llama-3-70B"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">895&#x202F;&#x219B;&#x202F;896</span><span class="annotate long">line 895 didn't jump to line 896</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t896" href="#t896">896</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t897" href="#t897">897</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t898" href="#t898">898</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t899" href="#t899">899</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">64</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t900" href="#t900">900</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">28672</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t901" href="#t901">901</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">80</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t902" href="#t902">902</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t903" href="#t903">903</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t904" href="#t904">904</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">128256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t905" href="#t905">905</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t906" href="#t906">906</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t907" href="#t907">907</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t908" href="#t908">908</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t909" href="#t909">909</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t910" href="#t910">910</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t911" href="#t911">911</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t912" href="#t912">912</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t913" href="#t913">913</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">500000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t914" href="#t914">914</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t915" href="#t915">915</a></span><span class="t">    <span class="key">elif</span> <span class="str">"Llama-3.2-1B"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">915&#x202F;&#x219B;&#x202F;916</span><span class="annotate long">line 915 didn't jump to line 916</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t916" href="#t916">916</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t917" href="#t917">917</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t918" href="#t918">918</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">64</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t919" href="#t919">919</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t920" href="#t920">920</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t921" href="#t921">921</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">16</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t922" href="#t922">922</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>  <span class="com"># capped due to memory issues</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t923" href="#t923">923</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t924" href="#t924">924</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">128256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t925" href="#t925">925</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t926" href="#t926">926</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t927" href="#t927">927</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t928" href="#t928">928</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t929" href="#t929">929</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t930" href="#t930">930</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">64</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t931" href="#t931">931</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t932" href="#t932">932</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t933" href="#t933">933</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">500000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t934" href="#t934">934</a></span><span class="t">            <span class="str">"use_NTK_by_parts_rope"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t935" href="#t935">935</a></span><span class="t">            <span class="str">"NTK_by_parts_low_freq_factor"</span><span class="op">:</span> <span class="num">1.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t936" href="#t936">936</a></span><span class="t">            <span class="str">"NTK_by_parts_high_freq_factor"</span><span class="op">:</span> <span class="num">4.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t937" href="#t937">937</a></span><span class="t">            <span class="str">"NTK_by_parts_factor"</span><span class="op">:</span> <span class="num">32.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t938" href="#t938">938</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t939" href="#t939">939</a></span><span class="t">    <span class="key">elif</span> <span class="str">"Llama-3.2-3B"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">939&#x202F;&#x219B;&#x202F;940</span><span class="annotate long">line 939 didn't jump to line 940</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t940" href="#t940">940</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t941" href="#t941">941</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">3072</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t942" href="#t942">942</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t943" href="#t943">943</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">24</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t944" href="#t944">944</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t945" href="#t945">945</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">28</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t946" href="#t946">946</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>  <span class="com"># capped due to memory issues</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t947" href="#t947">947</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t948" href="#t948">948</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">128256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t949" href="#t949">949</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t950" href="#t950">950</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t951" href="#t951">951</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t952" href="#t952">952</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t953" href="#t953">953</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t954" href="#t954">954</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t955" href="#t955">955</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t956" href="#t956">956</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t957" href="#t957">957</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">500000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t958" href="#t958">958</a></span><span class="t">            <span class="str">"use_NTK_by_parts_rope"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t959" href="#t959">959</a></span><span class="t">            <span class="str">"NTK_by_parts_low_freq_factor"</span><span class="op">:</span> <span class="num">1.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t960" href="#t960">960</a></span><span class="t">            <span class="str">"NTK_by_parts_high_freq_factor"</span><span class="op">:</span> <span class="num">4.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t961" href="#t961">961</a></span><span class="t">            <span class="str">"NTK_by_parts_factor"</span><span class="op">:</span> <span class="num">32.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t962" href="#t962">962</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t963" href="#t963">963</a></span><span class="t">    <span class="key">elif</span> <span class="str">"Llama-3.1-8B"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">963&#x202F;&#x219B;&#x202F;964</span><span class="annotate long">line 963 didn't jump to line 964</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t964" href="#t964">964</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t965" href="#t965">965</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t966" href="#t966">966</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t967" href="#t967">967</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t968" href="#t968">968</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">14336</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t969" href="#t969">969</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t970" href="#t970">970</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>  <span class="com"># capped due to memory issues</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t971" href="#t971">971</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t972" href="#t972">972</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">128256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t973" href="#t973">973</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t974" href="#t974">974</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t975" href="#t975">975</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t976" href="#t976">976</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t977" href="#t977">977</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t978" href="#t978">978</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t979" href="#t979">979</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t980" href="#t980">980</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t981" href="#t981">981</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">500000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t982" href="#t982">982</a></span><span class="t">            <span class="str">"use_NTK_by_parts_rope"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t983" href="#t983">983</a></span><span class="t">            <span class="str">"NTK_by_parts_low_freq_factor"</span><span class="op">:</span> <span class="num">1.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t984" href="#t984">984</a></span><span class="t">            <span class="str">"NTK_by_parts_high_freq_factor"</span><span class="op">:</span> <span class="num">4.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t985" href="#t985">985</a></span><span class="t">            <span class="str">"NTK_by_parts_factor"</span><span class="op">:</span> <span class="num">8.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t986" href="#t986">986</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t987" href="#t987">987</a></span><span class="t">    <span class="key">elif</span> <span class="str">"Llama-3.1-70B"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">987&#x202F;&#x219B;&#x202F;988</span><span class="annotate long">line 987 didn't jump to line 988</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t988" href="#t988">988</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t989" href="#t989">989</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t990" href="#t990">990</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t991" href="#t991">991</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">64</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t992" href="#t992">992</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">28672</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t993" href="#t993">993</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">80</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t994" href="#t994">994</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>  <span class="com"># capped due to memory issues</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t995" href="#t995">995</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t996" href="#t996">996</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">128256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t997" href="#t997">997</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t998" href="#t998">998</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t999" href="#t999">999</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1000" href="#t1000">1000</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1001" href="#t1001">1001</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1002" href="#t1002">1002</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1003" href="#t1003">1003</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1004" href="#t1004">1004</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1005" href="#t1005">1005</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">500000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1006" href="#t1006">1006</a></span><span class="t">            <span class="str">"use_NTK_by_parts_rope"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1007" href="#t1007">1007</a></span><span class="t">            <span class="str">"NTK_by_parts_low_freq_factor"</span><span class="op">:</span> <span class="num">1.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1008" href="#t1008">1008</a></span><span class="t">            <span class="str">"NTK_by_parts_high_freq_factor"</span><span class="op">:</span> <span class="num">4.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1009" href="#t1009">1009</a></span><span class="t">            <span class="str">"NTK_by_parts_factor"</span><span class="op">:</span> <span class="num">8.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1010" href="#t1010">1010</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1011" href="#t1011">1011</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"GPTNeoForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1012" href="#t1012">1012</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1013" href="#t1013">1013</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1014" href="#t1014">1014</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1015" href="#t1015">1015</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1016" href="#t1016">1016</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">*</span> <span class="num">4</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1017" href="#t1017">1017</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1018" href="#t1018">1018</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">max_position_embeddings</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1019" href="#t1019">1019</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">layer_norm_epsilon</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1020" href="#t1020">1020</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1021" href="#t1021">1021</a></span><span class="t">            <span class="str">"attn_types"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">attention_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1022" href="#t1022">1022</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">activation_function</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1023" href="#t1023">1023</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1024" href="#t1024">1024</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1025" href="#t1025">1025</a></span><span class="t">            <span class="str">"window_size"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">window_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1026" href="#t1026">1026</a></span><span class="t">            <span class="str">"scale_attn_by_inverse_layer_idx"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1027" href="#t1027">1027</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"LN"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1028" href="#t1028">1028</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1029" href="#t1029">1029</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"GPT2LMHeadModel"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1030" href="#t1030">1030</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1031" href="#t1031">1031</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_embd</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1032" href="#t1032">1032</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_embd</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1033" href="#t1033">1033</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1034" href="#t1034">1034</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_embd</span> <span class="op">*</span> <span class="num">4</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1035" href="#t1035">1035</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_layer</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1036" href="#t1036">1036</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_ctx</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1037" href="#t1037">1037</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">layer_norm_epsilon</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1038" href="#t1038">1038</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1039" href="#t1039">1039</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">activation_function</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1040" href="#t1040">1040</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1041" href="#t1041">1041</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1042" href="#t1042">1042</a></span><span class="t">            <span class="str">"scale_attn_by_inverse_layer_idx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">scale_attn_by_inverse_layer_idx</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1043" href="#t1043">1043</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"LN"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1044" href="#t1044">1044</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1045" href="#t1045">1045</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"OPTForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1046" href="#t1046">1046</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1047" href="#t1047">1047</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1048" href="#t1048">1048</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1049" href="#t1049">1049</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1050" href="#t1050">1050</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">ffn_dim</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1051" href="#t1051">1051</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1052" href="#t1052">1052</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">max_position_embeddings</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1053" href="#t1053">1053</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1054" href="#t1054">1054</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1055" href="#t1055">1055</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">activation_function</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1056" href="#t1056">1056</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1057" href="#t1057">1057</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1058" href="#t1058">1058</a></span><span class="t">            <span class="str">"scale_attn_by_inverse_layer_idx"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1059" href="#t1059">1059</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"LN"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1060" href="#t1060">1060</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1061" href="#t1061">1061</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"GPTJForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1062" href="#t1062">1062</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1063" href="#t1063">1063</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_embd</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1064" href="#t1064">1064</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_embd</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1065" href="#t1065">1065</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1066" href="#t1066">1066</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">4</span> <span class="op">*</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_embd</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1067" href="#t1067">1067</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_layer</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1068" href="#t1068">1068</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_positions</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1069" href="#t1069">1069</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1070" href="#t1070">1070</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1071" href="#t1071">1071</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">activation_function</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1072" href="#t1072">1072</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1073" href="#t1073">1073</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1074" href="#t1074">1074</a></span><span class="t">            <span class="str">"scale_attn_by_inverse_layer_idx"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1075" href="#t1075">1075</a></span><span class="t">            <span class="str">"parallel_attn_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1076" href="#t1076">1076</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1077" href="#t1077">1077</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rotary_dim</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1078" href="#t1078">1078</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1079" href="#t1079">1079</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"LN"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1080" href="#t1080">1080</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1081" href="#t1081">1081</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"GPTNeoXForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1082" href="#t1082">1082</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1083" href="#t1083">1083</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1084" href="#t1084">1084</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1085" href="#t1085">1085</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1086" href="#t1086">1086</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">intermediate_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1087" href="#t1087">1087</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1088" href="#t1088">1088</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">max_position_embeddings</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1089" href="#t1089">1089</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">layer_norm_eps</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1090" href="#t1090">1090</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1091" href="#t1091">1091</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_act</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1092" href="#t1092">1092</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1093" href="#t1093">1093</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1094" href="#t1094">1094</a></span><span class="t">            <span class="str">"scale_attn_by_inverse_layer_idx"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1095" href="#t1095">1095</a></span><span class="t">            <span class="str">"parallel_attn_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1096" href="#t1096">1096</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1097" href="#t1097">1097</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1098" href="#t1098">1098</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"LN"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1099" href="#t1099">1099</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1100" href="#t1100">1100</a></span><span class="t">        <span class="nam">rotary_pct</span> <span class="op">=</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rotary_pct</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1101" href="#t1101">1101</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"rotary_dim"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">round</span><span class="op">(</span><span class="nam">rotary_pct</span> <span class="op">*</span> <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"d_head"</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1102" href="#t1102">1102</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"BertForMaskedLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1103" href="#t1103">1103</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1104" href="#t1104">1104</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1105" href="#t1105">1105</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1106" href="#t1106">1106</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1107" href="#t1107">1107</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">intermediate_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1108" href="#t1108">1108</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1109" href="#t1109">1109</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">max_position_embeddings</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1110" href="#t1110">1110</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">layer_norm_eps</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1111" href="#t1111">1111</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1112" href="#t1112">1112</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"gelu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1113" href="#t1113">1113</a></span><span class="t">            <span class="str">"attention_dir"</span><span class="op">:</span> <span class="str">"bidirectional"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1114" href="#t1114">1114</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1115" href="#t1115">1115</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"MistralForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1115&#x202F;&#x219B;&#x202F;1116</span><span class="annotate long">line 1115 didn't jump to line 1116, because the condition on line 1115 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1116" href="#t1116">1116</a></span><span class="t">        <span class="nam">use_local_attn</span> <span class="op">=</span> <span class="key">True</span> <span class="key">if</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">sliding_window</span> <span class="key">else</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1117" href="#t1117">1117</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1118" href="#t1118">1118</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1119" href="#t1119">1119</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">head_dim</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1120" href="#t1120">1120</a></span><span class="t">            <span class="key">if</span> <span class="nam">hasattr</span><span class="op">(</span><span class="nam">hf_config</span><span class="op">,</span> <span class="str">"head_dim"</span><span class="op">)</span> <span class="key">and</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">head_dim</span> <span class="op">></span> <span class="num">0</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1121" href="#t1121">1121</a></span><span class="t">            <span class="key">else</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1122" href="#t1122">1122</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1123" href="#t1123">1123</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">intermediate_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1124" href="#t1124">1124</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1125" href="#t1125">1125</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>  <span class="com"># Capped due to memory issues</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1126" href="#t1126">1126</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1127" href="#t1127">1127</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_act</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1128" href="#t1128">1128</a></span><span class="t">            <span class="str">"window_size"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">sliding_window</span><span class="op">,</span>  <span class="com"># None if no sliding window was used</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1129" href="#t1129">1129</a></span><span class="t">            <span class="str">"attn_types"</span><span class="op">:</span> <span class="op">[</span><span class="str">"local"</span><span class="op">]</span> <span class="op">*</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span> <span class="key">if</span> <span class="nam">use_local_attn</span> <span class="key">else</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1130" href="#t1130">1130</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rms_norm_eps</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1131" href="#t1131">1131</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rope_theta</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1132" href="#t1132">1132</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_key_value_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1133" href="#t1133">1133</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="nam">use_local_attn</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1134" href="#t1134">1134</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1135" href="#t1135">1135</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1136" href="#t1136">1136</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1137" href="#t1137">1137</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1138" href="#t1138">1138</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"MixtralForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1138&#x202F;&#x219B;&#x202F;1139</span><span class="annotate long">line 1138 didn't jump to line 1139</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1139" href="#t1139">1139</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1140" href="#t1140">1140</a></span><span class="t">            <span class="str">"dtype"</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">bfloat16</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1141" href="#t1141">1141</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1142" href="#t1142">1142</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1143" href="#t1143">1143</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1144" href="#t1144">1144</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">intermediate_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1145" href="#t1145">1145</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1146" href="#t1146">1146</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">max_position_embeddings</span><span class="op">,</span>  <span class="com"># Capped due to memory issues</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1147" href="#t1147">1147</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1148" href="#t1148">1148</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_act</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1149" href="#t1149">1149</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1150" href="#t1150">1150</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1151" href="#t1151">1151</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rope_theta</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1152" href="#t1152">1152</a></span><span class="t">            <span class="str">"window_size"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">sliding_window</span><span class="op">,</span>  <span class="com"># This is None, as no sliding window was used</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1153" href="#t1153">1153</a></span><span class="t">            <span class="str">"attn_types"</span><span class="op">:</span> <span class="op">[</span><span class="str">"global"</span><span class="op">]</span> <span class="op">*</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1154" href="#t1154">1154</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rms_norm_eps</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1155" href="#t1155">1155</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_key_value_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1156" href="#t1156">1156</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1157" href="#t1157">1157</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1158" href="#t1158">1158</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1159" href="#t1159">1159</a></span><span class="t">            <span class="str">"num_experts"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_local_experts</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1160" href="#t1160">1160</a></span><span class="t">            <span class="str">"experts_per_token"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_experts_per_tok</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1161" href="#t1161">1161</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1162" href="#t1162">1162</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"BloomForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1163" href="#t1163">1163</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1164" href="#t1164">1164</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1165" href="#t1165">1165</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1166" href="#t1166">1166</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1167" href="#t1167">1167</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">*</span> <span class="num">4</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1168" href="#t1168">1168</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_layer</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1169" href="#t1169">1169</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>  <span class="com"># Capped due to HF Tokenizer Constraints</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1170" href="#t1170">1170</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1171" href="#t1171">1171</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"gelu_fast"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1172" href="#t1172">1172</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">layer_norm_epsilon</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1173" href="#t1173">1173</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"LN"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1174" href="#t1174">1174</a></span><span class="t">            <span class="str">"post_embedding_ln"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1175" href="#t1175">1175</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"alibi"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1176" href="#t1176">1176</a></span><span class="t">            <span class="str">"default_prepend_bos"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1177" href="#t1177">1177</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1178" href="#t1178">1178</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"GPT2LMHeadCustomModel"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1178&#x202F;&#x219B;&#x202F;1180</span><span class="annotate long">line 1178 didn't jump to line 1180</span></span></p>
-    <p class="pln"><span class="n"><a id="t1179" href="#t1179">1179</a></span><span class="t">        <span class="com"># santacoder</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1180" href="#t1180">1180</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1181" href="#t1181">1181</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_embd</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1182" href="#t1182">1182</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_embd</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1183" href="#t1183">1183</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1184" href="#t1184">1184</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_embd</span> <span class="op">*</span> <span class="num">4</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1185" href="#t1185">1185</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_layer</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1186" href="#t1186">1186</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_positions</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1187" href="#t1187">1187</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">layer_norm_epsilon</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1188" href="#t1188">1188</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1189" href="#t1189">1189</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">activation_function</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1190" href="#t1190">1190</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1191" href="#t1191">1191</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1192" href="#t1192">1192</a></span><span class="t">            <span class="str">"trust_remote_code"</span><span class="op">:</span> <span class="str">"santacoder"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1193" href="#t1193">1193</a></span><span class="t">            <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">,</span>  <span class="com"># Only santacoder needs trust_remote_code</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1194" href="#t1194">1194</a></span><span class="t">            <span class="str">"scale_attn_by_inverse_layer_idx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">scale_attn_by_inverse_layer_idx</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1195" href="#t1195">1195</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"LN"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1196" href="#t1196">1196</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1197" href="#t1197">1197</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"LlamaForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1197&#x202F;&#x219B;&#x202F;1198</span><span class="annotate long">line 1197 didn't jump to line 1198</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1198" href="#t1198">1198</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1199" href="#t1199">1199</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1200" href="#t1200">1200</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1201" href="#t1201">1201</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1202" href="#t1202">1202</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">intermediate_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1203" href="#t1203">1203</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1204" href="#t1204">1204</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">max_position_embeddings</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1205" href="#t1205">1205</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rms_norm_eps</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1206" href="#t1206">1206</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1207" href="#t1207">1207</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_act</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1208" href="#t1208">1208</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1209" href="#t1209">1209</a></span><span class="t">                <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_key_value_heads</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1210" href="#t1210">1210</a></span><span class="t">                <span class="key">if</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_key_value_heads</span> <span class="op">!=</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1211" href="#t1211">1211</a></span><span class="t">                <span class="key">else</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1212" href="#t1212">1212</a></span><span class="t">            <span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1213" href="#t1213">1213</a></span><span class="t">            <span class="com"># This is done because the current implementation of GQA will use Grouped-Query Attention if</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1214" href="#t1214">1214</a></span><span class="t">            <span class="com"># n_key_value_heads is not None, but hf_config.num_key_value_heads is sometimes specified as</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1215" href="#t1215">1215</a></span><span class="t">            <span class="com"># the same as hf_config.num_attention_heads, in which case GQA should not be used.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1216" href="#t1216">1216</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1217" href="#t1217">1217</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1218" href="#t1218">1218</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1219" href="#t1219">1219</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1220" href="#t1220">1220</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1221" href="#t1221">1221</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1222" href="#t1222">1222</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1223" href="#t1223">1223</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"QWenLMHeadModel"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1223&#x202F;&#x219B;&#x202F;1224</span><span class="annotate long">line 1223 didn't jump to line 1224</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1224" href="#t1224">1224</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1225" href="#t1225">1225</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1226" href="#t1226">1226</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1227" href="#t1227">1227</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1228" href="#t1228">1228</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">intermediate_size</span> <span class="op">//</span> <span class="num">2</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1229" href="#t1229">1229</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1230" href="#t1230">1230</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>  <span class="com"># Capped bc the actual ctx length is 30k and the attn mask would be too big</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1231" href="#t1231">1231</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">layer_norm_epsilon</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1232" href="#t1232">1232</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1233" href="#t1233">1233</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1234" href="#t1234">1234</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">scale_attn_weights</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1235" href="#t1235">1235</a></span><span class="t">            <span class="str">"initializer_range"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">initializer_range</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1236" href="#t1236">1236</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1237" href="#t1237">1237</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1238" href="#t1238">1238</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">kv_channels</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1239" href="#t1239">1239</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1240" href="#t1240">1240</a></span><span class="t">            <span class="str">"tokenizer_prepends_bos"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1241" href="#t1241">1241</a></span><span class="t">            <span class="str">"trust_remote_code"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1242" href="#t1242">1242</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1243" href="#t1243">1243</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1244" href="#t1244">1244</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1245" href="#t1245">1245</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"Qwen2ForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1245&#x202F;&#x219B;&#x202F;1247</span><span class="annotate long">line 1245 didn't jump to line 1247</span></span></p>
-    <p class="pln"><span class="n"><a id="t1246" href="#t1246">1246</a></span><span class="t">        <span class="com"># Note that Qwen1.5 models have architecture type Qwen2ForCausalLM.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1247" href="#t1247">1247</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1248" href="#t1248">1248</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1249" href="#t1249">1249</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1250" href="#t1250">1250</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1251" href="#t1251">1251</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_key_value_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1252" href="#t1252">1252</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">intermediate_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1253" href="#t1253">1253</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1254" href="#t1254">1254</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>  <span class="com"># Capped bc the actual ctx length is 30k and the attn mask would be too big</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1255" href="#t1255">1255</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rms_norm_eps</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1256" href="#t1256">1256</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1257" href="#t1257">1257</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_act</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1258" href="#t1258">1258</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1259" href="#t1259">1259</a></span><span class="t">            <span class="str">"initializer_range"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">initializer_range</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1260" href="#t1260">1260</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1261" href="#t1261">1261</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1262" href="#t1262">1262</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rope_theta</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1263" href="#t1263">1263</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1264" href="#t1264">1264</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t729" href="#t729">729</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t730" href="#t730">730</a></span><span class="t"><span class="key">def</span> <span class="nam">convert_hf_model_config</span><span class="op">(</span><span class="nam">model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t731" href="#t731">731</a></span><span class="t">    <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t732" href="#t732">732</a></span><span class="t"><span class="str">    Returns the model config for a HuggingFace model, converted to a dictionary</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t733" href="#t733">733</a></span><span class="t"><span class="str">    in the HookedTransformerConfig format.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t734" href="#t734">734</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t735" href="#t735">735</a></span><span class="t"><span class="str">    Takes the official_model_name as an input.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t736" href="#t736">736</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t737" href="#t737">737</a></span><span class="t">    <span class="com"># In case the user passed in an alias</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t738" href="#t738">738</a></span><span class="t">    <span class="key">if</span> <span class="op">(</span><span class="nam">Path</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span> <span class="op">/</span> <span class="str">"config.json"</span><span class="op">)</span><span class="op">.</span><span class="nam">exists</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">738&#x202F;&#x219B;&#x202F;739</span><span class="annotate long">line 738 didn't jump to line 739, because the condition on line 738 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t739" href="#t739">739</a></span><span class="t">        <span class="nam">logging</span><span class="op">.</span><span class="nam">info</span><span class="op">(</span><span class="str">"Loading model config from local directory"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t740" href="#t740">740</a></span><span class="t">        <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">model_name</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t741" href="#t741">741</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t742" href="#t742">742</a></span><span class="t">        <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">get_official_model_name</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t743" href="#t743">743</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t744" href="#t744">744</a></span><span class="t">    <span class="com"># Load HuggingFace model config</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t745" href="#t745">745</a></span><span class="t">    <span class="key">if</span> <span class="str">"llama"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">745&#x202F;&#x219B;&#x202F;746</span><span class="annotate long">line 745 didn't jump to line 746, because the condition on line 745 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t746" href="#t746">746</a></span><span class="t">        <span class="nam">architecture</span> <span class="op">=</span> <span class="str">"LlamaForCausalLM"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t747" href="#t747">747</a></span><span class="t">    <span class="key">elif</span> <span class="str">"gemma-2"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">747&#x202F;&#x219B;&#x202F;748</span><span class="annotate long">line 747 didn't jump to line 748, because the condition on line 747 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t748" href="#t748">748</a></span><span class="t">        <span class="nam">architecture</span> <span class="op">=</span> <span class="str">"Gemma2ForCausalLM"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t749" href="#t749">749</a></span><span class="t">    <span class="key">elif</span> <span class="str">"gemma"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">749&#x202F;&#x219B;&#x202F;750</span><span class="annotate long">line 749 didn't jump to line 750, because the condition on line 749 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t750" href="#t750">750</a></span><span class="t">        <span class="nam">architecture</span> <span class="op">=</span> <span class="str">"GemmaForCausalLM"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t751" href="#t751">751</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t752" href="#t752">752</a></span><span class="t">        <span class="nam">huggingface_token</span> <span class="op">=</span> <span class="nam">os</span><span class="op">.</span><span class="nam">environ</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"HF_TOKEN"</span><span class="op">,</span> <span class="key">None</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t753" href="#t753">753</a></span><span class="t">        <span class="nam">hf_config</span> <span class="op">=</span> <span class="nam">AutoConfig</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t754" href="#t754">754</a></span><span class="t">            <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t755" href="#t755">755</a></span><span class="t">            <span class="nam">token</span><span class="op">=</span><span class="nam">huggingface_token</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t756" href="#t756">756</a></span><span class="t">            <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t757" href="#t757">757</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t758" href="#t758">758</a></span><span class="t">        <span class="nam">architecture</span> <span class="op">=</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">architectures</span><span class="op">[</span><span class="num">0</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t759" href="#t759">759</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t760" href="#t760">760</a></span><span class="t">    <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span>&nbsp;</span><span class="r"><span class="annotate short">760&#x202F;&#x219B;&#x202F;763</span><span class="annotate long">line 760 didn't jump to line 763</span></span></p>
+    <p class="pln"><span class="n"><a id="t761" href="#t761">761</a></span><span class="t">        <span class="op">(</span><span class="str">"llama-7b"</span><span class="op">,</span> <span class="str">"meta-llama/Llama-2-7b"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t762" href="#t762">762</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>  <span class="com"># same architecture for LLaMA and Llama-2</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t763" href="#t763">763</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t764" href="#t764">764</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t765" href="#t765">765</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">4096</span> <span class="op">//</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t766" href="#t766">766</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t767" href="#t767">767</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">11008</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t768" href="#t768">768</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t769" href="#t769">769</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span> <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"llama-7b"</span><span class="op">)</span> <span class="key">else</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t770" href="#t770">770</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-6</span> <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"llama-7b"</span><span class="op">)</span> <span class="key">else</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t771" href="#t771">771</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">32000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t772" href="#t772">772</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t773" href="#t773">773</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t774" href="#t774">774</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t775" href="#t775">775</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t776" href="#t776">776</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">4096</span> <span class="op">//</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t777" href="#t777">777</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t778" href="#t778">778</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t779" href="#t779">779</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t780" href="#t780">780</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"codellama"</span><span class="op">)</span><span class="op">:</span>  <span class="com"># same architecture CodeLlama and Llama-2</span>&nbsp;</span><span class="r"><span class="annotate short">780&#x202F;&#x219B;&#x202F;781</span><span class="annotate long">line 780 didn't jump to line 781</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t781" href="#t781">781</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t782" href="#t782">782</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t783" href="#t783">783</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">4096</span> <span class="op">//</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t784" href="#t784">784</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t785" href="#t785">785</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">11008</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t786" href="#t786">786</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t787" href="#t787">787</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t788" href="#t788">788</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t789" href="#t789">789</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">32016</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t790" href="#t790">790</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t791" href="#t791">791</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t792" href="#t792">792</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t793" href="#t793">793</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">4096</span> <span class="op">//</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t794" href="#t794">794</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t795" href="#t795">795</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t796" href="#t796">796</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">1000000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t797" href="#t797">797</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t798" href="#t798">798</a></span><span class="t">        <span class="key">if</span> <span class="str">"python"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t799" href="#t799">799</a></span><span class="t">            <span class="com"># The vocab size of python version of CodeLlama-7b is 32000</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t800" href="#t800">800</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"d_vocab"</span><span class="op">]</span> <span class="op">=</span> <span class="num">32000</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t801" href="#t801">801</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span>&nbsp;</span><span class="r"><span class="annotate short">801&#x202F;&#x219B;&#x202F;804</span><span class="annotate long">line 801 didn't jump to line 804</span></span></p>
+    <p class="pln"><span class="n"><a id="t802" href="#t802">802</a></span><span class="t">        <span class="op">(</span><span class="str">"llama-13b"</span><span class="op">,</span> <span class="str">"meta-llama/Llama-2-13b"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t803" href="#t803">803</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>  <span class="com"># same architecture for LLaMA and Llama-2</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t804" href="#t804">804</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t805" href="#t805">805</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">5120</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t806" href="#t806">806</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">5120</span> <span class="op">//</span> <span class="num">40</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t807" href="#t807">807</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">40</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t808" href="#t808">808</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">13824</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t809" href="#t809">809</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">40</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t810" href="#t810">810</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span> <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"llama-13b"</span><span class="op">)</span> <span class="key">else</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t811" href="#t811">811</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-6</span> <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"llama-13b"</span><span class="op">)</span> <span class="key">else</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t812" href="#t812">812</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">32000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t813" href="#t813">813</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t814" href="#t814">814</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t815" href="#t815">815</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t816" href="#t816">816</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t817" href="#t817">817</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">5120</span> <span class="op">//</span> <span class="num">40</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t818" href="#t818">818</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t819" href="#t819">819</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t820" href="#t820">820</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t821" href="#t821">821</a></span><span class="t">    <span class="key">elif</span> <span class="str">"llama-30b"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">821&#x202F;&#x219B;&#x202F;822</span><span class="annotate long">line 821 didn't jump to line 822</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t822" href="#t822">822</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t823" href="#t823">823</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">6656</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t824" href="#t824">824</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">6656</span> <span class="op">//</span> <span class="num">52</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t825" href="#t825">825</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">52</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t826" href="#t826">826</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">17920</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t827" href="#t827">827</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">60</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t828" href="#t828">828</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t829" href="#t829">829</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-6</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t830" href="#t830">830</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">32000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t831" href="#t831">831</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t832" href="#t832">832</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t833" href="#t833">833</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t834" href="#t834">834</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t835" href="#t835">835</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">6656</span> <span class="op">//</span> <span class="num">52</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t836" href="#t836">836</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t837" href="#t837">837</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t838" href="#t838">838</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t839" href="#t839">839</a></span><span class="t">    <span class="key">elif</span> <span class="str">"llama-65b"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">839&#x202F;&#x219B;&#x202F;840</span><span class="annotate long">line 839 didn't jump to line 840</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t840" href="#t840">840</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t841" href="#t841">841</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t842" href="#t842">842</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">8192</span> <span class="op">//</span> <span class="num">64</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t843" href="#t843">843</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">64</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t844" href="#t844">844</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">22016</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t845" href="#t845">845</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">80</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t846" href="#t846">846</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t847" href="#t847">847</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-6</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t848" href="#t848">848</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">32000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t849" href="#t849">849</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t850" href="#t850">850</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t851" href="#t851">851</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t852" href="#t852">852</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">8192</span> <span class="op">//</span> <span class="num">64</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t853" href="#t853">853</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t854" href="#t854">854</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t855" href="#t855">855</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t856" href="#t856">856</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t857" href="#t857">857</a></span><span class="t">    <span class="key">elif</span> <span class="str">"Llama-2-70b"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">857&#x202F;&#x219B;&#x202F;858</span><span class="annotate long">line 857 didn't jump to line 858</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t858" href="#t858">858</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t859" href="#t859">859</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t860" href="#t860">860</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t861" href="#t861">861</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">64</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t862" href="#t862">862</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">28672</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t863" href="#t863">863</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">80</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t864" href="#t864">864</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t865" href="#t865">865</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t866" href="#t866">866</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">32000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t867" href="#t867">867</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t868" href="#t868">868</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t869" href="#t869">869</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t870" href="#t870">870</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t871" href="#t871">871</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t872" href="#t872">872</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t873" href="#t873">873</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t874" href="#t874">874</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t875" href="#t875">875</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t876" href="#t876">876</a></span><span class="t">    <span class="key">elif</span> <span class="str">"Meta-Llama-3-8B"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">876&#x202F;&#x219B;&#x202F;877</span><span class="annotate long">line 876 didn't jump to line 877</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t877" href="#t877">877</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t878" href="#t878">878</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t879" href="#t879">879</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t880" href="#t880">880</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t881" href="#t881">881</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">14336</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t882" href="#t882">882</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t883" href="#t883">883</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t884" href="#t884">884</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t885" href="#t885">885</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">128256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t886" href="#t886">886</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t887" href="#t887">887</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t888" href="#t888">888</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t889" href="#t889">889</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t890" href="#t890">890</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t891" href="#t891">891</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t892" href="#t892">892</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t893" href="#t893">893</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t894" href="#t894">894</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">500000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t895" href="#t895">895</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t896" href="#t896">896</a></span><span class="t">    <span class="key">elif</span> <span class="str">"Meta-Llama-3-70B"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">896&#x202F;&#x219B;&#x202F;897</span><span class="annotate long">line 896 didn't jump to line 897</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t897" href="#t897">897</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t898" href="#t898">898</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t899" href="#t899">899</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t900" href="#t900">900</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">64</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t901" href="#t901">901</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">28672</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t902" href="#t902">902</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">80</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t903" href="#t903">903</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t904" href="#t904">904</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t905" href="#t905">905</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">128256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t906" href="#t906">906</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t907" href="#t907">907</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t908" href="#t908">908</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t909" href="#t909">909</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t910" href="#t910">910</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t911" href="#t911">911</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t912" href="#t912">912</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t913" href="#t913">913</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t914" href="#t914">914</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">500000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t915" href="#t915">915</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t916" href="#t916">916</a></span><span class="t">    <span class="key">elif</span> <span class="str">"Llama-3.2-1B"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">916&#x202F;&#x219B;&#x202F;917</span><span class="annotate long">line 916 didn't jump to line 917</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t917" href="#t917">917</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t918" href="#t918">918</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t919" href="#t919">919</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">64</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t920" href="#t920">920</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t921" href="#t921">921</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t922" href="#t922">922</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">16</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t923" href="#t923">923</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>  <span class="com"># capped due to memory issues</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t924" href="#t924">924</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t925" href="#t925">925</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">128256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t926" href="#t926">926</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t927" href="#t927">927</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t928" href="#t928">928</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t929" href="#t929">929</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t930" href="#t930">930</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t931" href="#t931">931</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">64</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t932" href="#t932">932</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t933" href="#t933">933</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t934" href="#t934">934</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">500000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t935" href="#t935">935</a></span><span class="t">            <span class="str">"use_NTK_by_parts_rope"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t936" href="#t936">936</a></span><span class="t">            <span class="str">"NTK_by_parts_low_freq_factor"</span><span class="op">:</span> <span class="num">1.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t937" href="#t937">937</a></span><span class="t">            <span class="str">"NTK_by_parts_high_freq_factor"</span><span class="op">:</span> <span class="num">4.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t938" href="#t938">938</a></span><span class="t">            <span class="str">"NTK_by_parts_factor"</span><span class="op">:</span> <span class="num">32.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t939" href="#t939">939</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t940" href="#t940">940</a></span><span class="t">    <span class="key">elif</span> <span class="str">"Llama-3.2-3B"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">940&#x202F;&#x219B;&#x202F;941</span><span class="annotate long">line 940 didn't jump to line 941</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t941" href="#t941">941</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t942" href="#t942">942</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">3072</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t943" href="#t943">943</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t944" href="#t944">944</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">24</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t945" href="#t945">945</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t946" href="#t946">946</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">28</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t947" href="#t947">947</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>  <span class="com"># capped due to memory issues</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t948" href="#t948">948</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t949" href="#t949">949</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">128256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t950" href="#t950">950</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t951" href="#t951">951</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t952" href="#t952">952</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t953" href="#t953">953</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t954" href="#t954">954</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t955" href="#t955">955</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t956" href="#t956">956</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t957" href="#t957">957</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t958" href="#t958">958</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">500000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t959" href="#t959">959</a></span><span class="t">            <span class="str">"use_NTK_by_parts_rope"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t960" href="#t960">960</a></span><span class="t">            <span class="str">"NTK_by_parts_low_freq_factor"</span><span class="op">:</span> <span class="num">1.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t961" href="#t961">961</a></span><span class="t">            <span class="str">"NTK_by_parts_high_freq_factor"</span><span class="op">:</span> <span class="num">4.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t962" href="#t962">962</a></span><span class="t">            <span class="str">"NTK_by_parts_factor"</span><span class="op">:</span> <span class="num">32.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t963" href="#t963">963</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t964" href="#t964">964</a></span><span class="t">    <span class="key">elif</span> <span class="str">"Llama-3.3-70B"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">964&#x202F;&#x219B;&#x202F;965</span><span class="annotate long">line 964 didn't jump to line 965</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t965" href="#t965">965</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t966" href="#t966">966</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t967" href="#t967">967</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t968" href="#t968">968</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">64</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t969" href="#t969">969</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">28672</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t970" href="#t970">970</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">80</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t971" href="#t971">971</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>  <span class="com"># capped due to memory issues</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t972" href="#t972">972</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t973" href="#t973">973</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">128256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t974" href="#t974">974</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t975" href="#t975">975</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t976" href="#t976">976</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t977" href="#t977">977</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t978" href="#t978">978</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t979" href="#t979">979</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t980" href="#t980">980</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t981" href="#t981">981</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t982" href="#t982">982</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">500000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t983" href="#t983">983</a></span><span class="t">            <span class="str">"use_NTK_by_parts_rope"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t984" href="#t984">984</a></span><span class="t">            <span class="str">"NTK_by_parts_low_freq_factor"</span><span class="op">:</span> <span class="num">1.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t985" href="#t985">985</a></span><span class="t">            <span class="str">"NTK_by_parts_high_freq_factor"</span><span class="op">:</span> <span class="num">4.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t986" href="#t986">986</a></span><span class="t">            <span class="str">"NTK_by_parts_factor"</span><span class="op">:</span> <span class="num">8.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t987" href="#t987">987</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t988" href="#t988">988</a></span><span class="t">    <span class="key">elif</span> <span class="str">"Llama-3.1-8B"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">988&#x202F;&#x219B;&#x202F;989</span><span class="annotate long">line 988 didn't jump to line 989</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t989" href="#t989">989</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t990" href="#t990">990</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t991" href="#t991">991</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t992" href="#t992">992</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t993" href="#t993">993</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">14336</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t994" href="#t994">994</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t995" href="#t995">995</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>  <span class="com"># capped due to memory issues</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t996" href="#t996">996</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t997" href="#t997">997</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">128256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t998" href="#t998">998</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t999" href="#t999">999</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1000" href="#t1000">1000</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1001" href="#t1001">1001</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1002" href="#t1002">1002</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1003" href="#t1003">1003</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1004" href="#t1004">1004</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1005" href="#t1005">1005</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1006" href="#t1006">1006</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">500000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1007" href="#t1007">1007</a></span><span class="t">            <span class="str">"use_NTK_by_parts_rope"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1008" href="#t1008">1008</a></span><span class="t">            <span class="str">"NTK_by_parts_low_freq_factor"</span><span class="op">:</span> <span class="num">1.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1009" href="#t1009">1009</a></span><span class="t">            <span class="str">"NTK_by_parts_high_freq_factor"</span><span class="op">:</span> <span class="num">4.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1010" href="#t1010">1010</a></span><span class="t">            <span class="str">"NTK_by_parts_factor"</span><span class="op">:</span> <span class="num">8.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1011" href="#t1011">1011</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1012" href="#t1012">1012</a></span><span class="t">    <span class="key">elif</span> <span class="str">"Llama-3.1-70B"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1012&#x202F;&#x219B;&#x202F;1013</span><span class="annotate long">line 1012 didn't jump to line 1013</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1013" href="#t1013">1013</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1014" href="#t1014">1014</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1015" href="#t1015">1015</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1016" href="#t1016">1016</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">64</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1017" href="#t1017">1017</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">28672</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1018" href="#t1018">1018</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">80</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1019" href="#t1019">1019</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>  <span class="com"># capped due to memory issues</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1020" href="#t1020">1020</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1021" href="#t1021">1021</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">128256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1022" href="#t1022">1022</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1023" href="#t1023">1023</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1024" href="#t1024">1024</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1025" href="#t1025">1025</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1026" href="#t1026">1026</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1027" href="#t1027">1027</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1028" href="#t1028">1028</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1029" href="#t1029">1029</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1030" href="#t1030">1030</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">500000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1031" href="#t1031">1031</a></span><span class="t">            <span class="str">"use_NTK_by_parts_rope"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1032" href="#t1032">1032</a></span><span class="t">            <span class="str">"NTK_by_parts_low_freq_factor"</span><span class="op">:</span> <span class="num">1.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1033" href="#t1033">1033</a></span><span class="t">            <span class="str">"NTK_by_parts_high_freq_factor"</span><span class="op">:</span> <span class="num">4.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1034" href="#t1034">1034</a></span><span class="t">            <span class="str">"NTK_by_parts_factor"</span><span class="op">:</span> <span class="num">8.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1035" href="#t1035">1035</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1036" href="#t1036">1036</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"GPTNeoForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1037" href="#t1037">1037</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1038" href="#t1038">1038</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1039" href="#t1039">1039</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1040" href="#t1040">1040</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1041" href="#t1041">1041</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">*</span> <span class="num">4</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1042" href="#t1042">1042</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1043" href="#t1043">1043</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">max_position_embeddings</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1044" href="#t1044">1044</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">layer_norm_epsilon</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1045" href="#t1045">1045</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1046" href="#t1046">1046</a></span><span class="t">            <span class="str">"attn_types"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">attention_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1047" href="#t1047">1047</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">activation_function</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1048" href="#t1048">1048</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1049" href="#t1049">1049</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1050" href="#t1050">1050</a></span><span class="t">            <span class="str">"window_size"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">window_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1051" href="#t1051">1051</a></span><span class="t">            <span class="str">"scale_attn_by_inverse_layer_idx"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1052" href="#t1052">1052</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"LN"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1053" href="#t1053">1053</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1054" href="#t1054">1054</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"GPT2LMHeadModel"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1055" href="#t1055">1055</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1056" href="#t1056">1056</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_embd</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1057" href="#t1057">1057</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_embd</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1058" href="#t1058">1058</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1059" href="#t1059">1059</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_embd</span> <span class="op">*</span> <span class="num">4</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1060" href="#t1060">1060</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_layer</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1061" href="#t1061">1061</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_ctx</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1062" href="#t1062">1062</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">layer_norm_epsilon</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1063" href="#t1063">1063</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1064" href="#t1064">1064</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">activation_function</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1065" href="#t1065">1065</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1066" href="#t1066">1066</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1067" href="#t1067">1067</a></span><span class="t">            <span class="str">"scale_attn_by_inverse_layer_idx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">scale_attn_by_inverse_layer_idx</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1068" href="#t1068">1068</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"LN"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1069" href="#t1069">1069</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1070" href="#t1070">1070</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"OPTForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1071" href="#t1071">1071</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1072" href="#t1072">1072</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1073" href="#t1073">1073</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1074" href="#t1074">1074</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1075" href="#t1075">1075</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">ffn_dim</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1076" href="#t1076">1076</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1077" href="#t1077">1077</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">max_position_embeddings</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1078" href="#t1078">1078</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1079" href="#t1079">1079</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1080" href="#t1080">1080</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">activation_function</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1081" href="#t1081">1081</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1082" href="#t1082">1082</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1083" href="#t1083">1083</a></span><span class="t">            <span class="str">"scale_attn_by_inverse_layer_idx"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1084" href="#t1084">1084</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"LN"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1085" href="#t1085">1085</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1086" href="#t1086">1086</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"GPTJForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1087" href="#t1087">1087</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1088" href="#t1088">1088</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_embd</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1089" href="#t1089">1089</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_embd</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1090" href="#t1090">1090</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1091" href="#t1091">1091</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">4</span> <span class="op">*</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_embd</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1092" href="#t1092">1092</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_layer</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1093" href="#t1093">1093</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_positions</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1094" href="#t1094">1094</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1095" href="#t1095">1095</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1096" href="#t1096">1096</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">activation_function</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1097" href="#t1097">1097</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1098" href="#t1098">1098</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1099" href="#t1099">1099</a></span><span class="t">            <span class="str">"scale_attn_by_inverse_layer_idx"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1100" href="#t1100">1100</a></span><span class="t">            <span class="str">"parallel_attn_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1101" href="#t1101">1101</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1102" href="#t1102">1102</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rotary_dim</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1103" href="#t1103">1103</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1104" href="#t1104">1104</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"LN"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1105" href="#t1105">1105</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1106" href="#t1106">1106</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"GPTNeoXForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1107" href="#t1107">1107</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1108" href="#t1108">1108</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1109" href="#t1109">1109</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1110" href="#t1110">1110</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1111" href="#t1111">1111</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">intermediate_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1112" href="#t1112">1112</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1113" href="#t1113">1113</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">max_position_embeddings</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1114" href="#t1114">1114</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">layer_norm_eps</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1115" href="#t1115">1115</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1116" href="#t1116">1116</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_act</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1117" href="#t1117">1117</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1118" href="#t1118">1118</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1119" href="#t1119">1119</a></span><span class="t">            <span class="str">"scale_attn_by_inverse_layer_idx"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1120" href="#t1120">1120</a></span><span class="t">            <span class="str">"parallel_attn_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1121" href="#t1121">1121</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1122" href="#t1122">1122</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1123" href="#t1123">1123</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"LN"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1124" href="#t1124">1124</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1125" href="#t1125">1125</a></span><span class="t">        <span class="nam">rotary_pct</span> <span class="op">=</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rotary_pct</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1126" href="#t1126">1126</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"rotary_dim"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">round</span><span class="op">(</span><span class="nam">rotary_pct</span> <span class="op">*</span> <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"d_head"</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1127" href="#t1127">1127</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"BertForMaskedLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1128" href="#t1128">1128</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1129" href="#t1129">1129</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1130" href="#t1130">1130</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1131" href="#t1131">1131</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1132" href="#t1132">1132</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">intermediate_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1133" href="#t1133">1133</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1134" href="#t1134">1134</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">max_position_embeddings</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1135" href="#t1135">1135</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">layer_norm_eps</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1136" href="#t1136">1136</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1137" href="#t1137">1137</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"gelu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1138" href="#t1138">1138</a></span><span class="t">            <span class="str">"attention_dir"</span><span class="op">:</span> <span class="str">"bidirectional"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1139" href="#t1139">1139</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1140" href="#t1140">1140</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"MistralForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1140&#x202F;&#x219B;&#x202F;1141</span><span class="annotate long">line 1140 didn't jump to line 1141, because the condition on line 1140 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1141" href="#t1141">1141</a></span><span class="t">        <span class="nam">use_local_attn</span> <span class="op">=</span> <span class="key">True</span> <span class="key">if</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">sliding_window</span> <span class="key">else</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1142" href="#t1142">1142</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1143" href="#t1143">1143</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1144" href="#t1144">1144</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">head_dim</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1145" href="#t1145">1145</a></span><span class="t">            <span class="key">if</span> <span class="nam">hasattr</span><span class="op">(</span><span class="nam">hf_config</span><span class="op">,</span> <span class="str">"head_dim"</span><span class="op">)</span> <span class="key">and</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">head_dim</span> <span class="op">></span> <span class="num">0</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1146" href="#t1146">1146</a></span><span class="t">            <span class="key">else</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1147" href="#t1147">1147</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1148" href="#t1148">1148</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">intermediate_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1149" href="#t1149">1149</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1150" href="#t1150">1150</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>  <span class="com"># Capped due to memory issues</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1151" href="#t1151">1151</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1152" href="#t1152">1152</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_act</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1153" href="#t1153">1153</a></span><span class="t">            <span class="str">"window_size"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">sliding_window</span><span class="op">,</span>  <span class="com"># None if no sliding window was used</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1154" href="#t1154">1154</a></span><span class="t">            <span class="str">"attn_types"</span><span class="op">:</span> <span class="op">[</span><span class="str">"local"</span><span class="op">]</span> <span class="op">*</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span> <span class="key">if</span> <span class="nam">use_local_attn</span> <span class="key">else</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1155" href="#t1155">1155</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rms_norm_eps</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1156" href="#t1156">1156</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rope_theta</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1157" href="#t1157">1157</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_key_value_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1158" href="#t1158">1158</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="nam">use_local_attn</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1159" href="#t1159">1159</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1160" href="#t1160">1160</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1161" href="#t1161">1161</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1162" href="#t1162">1162</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1163" href="#t1163">1163</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"MixtralForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1163&#x202F;&#x219B;&#x202F;1164</span><span class="annotate long">line 1163 didn't jump to line 1164</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1164" href="#t1164">1164</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1165" href="#t1165">1165</a></span><span class="t">            <span class="str">"dtype"</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">bfloat16</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1166" href="#t1166">1166</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1167" href="#t1167">1167</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1168" href="#t1168">1168</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1169" href="#t1169">1169</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">intermediate_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1170" href="#t1170">1170</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1171" href="#t1171">1171</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">max_position_embeddings</span><span class="op">,</span>  <span class="com"># Capped due to memory issues</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1172" href="#t1172">1172</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1173" href="#t1173">1173</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_act</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1174" href="#t1174">1174</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1175" href="#t1175">1175</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1176" href="#t1176">1176</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rope_theta</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1177" href="#t1177">1177</a></span><span class="t">            <span class="str">"window_size"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">sliding_window</span><span class="op">,</span>  <span class="com"># This is None, as no sliding window was used</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1178" href="#t1178">1178</a></span><span class="t">            <span class="str">"attn_types"</span><span class="op">:</span> <span class="op">[</span><span class="str">"global"</span><span class="op">]</span> <span class="op">*</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1179" href="#t1179">1179</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rms_norm_eps</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1180" href="#t1180">1180</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_key_value_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1181" href="#t1181">1181</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1182" href="#t1182">1182</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1183" href="#t1183">1183</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1184" href="#t1184">1184</a></span><span class="t">            <span class="str">"num_experts"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_local_experts</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1185" href="#t1185">1185</a></span><span class="t">            <span class="str">"experts_per_token"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_experts_per_tok</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1186" href="#t1186">1186</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1187" href="#t1187">1187</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"BloomForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1188" href="#t1188">1188</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1189" href="#t1189">1189</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1190" href="#t1190">1190</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1191" href="#t1191">1191</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1192" href="#t1192">1192</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">*</span> <span class="num">4</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1193" href="#t1193">1193</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_layer</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1194" href="#t1194">1194</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>  <span class="com"># Capped due to HF Tokenizer Constraints</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1195" href="#t1195">1195</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1196" href="#t1196">1196</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"gelu_fast"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1197" href="#t1197">1197</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">layer_norm_epsilon</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1198" href="#t1198">1198</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"LN"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1199" href="#t1199">1199</a></span><span class="t">            <span class="str">"post_embedding_ln"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1200" href="#t1200">1200</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"alibi"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1201" href="#t1201">1201</a></span><span class="t">            <span class="str">"default_prepend_bos"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1202" href="#t1202">1202</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1203" href="#t1203">1203</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"GPT2LMHeadCustomModel"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1203&#x202F;&#x219B;&#x202F;1205</span><span class="annotate long">line 1203 didn't jump to line 1205</span></span></p>
+    <p class="pln"><span class="n"><a id="t1204" href="#t1204">1204</a></span><span class="t">        <span class="com"># santacoder</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1205" href="#t1205">1205</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1206" href="#t1206">1206</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_embd</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1207" href="#t1207">1207</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_embd</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1208" href="#t1208">1208</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1209" href="#t1209">1209</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_embd</span> <span class="op">*</span> <span class="num">4</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1210" href="#t1210">1210</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_layer</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1211" href="#t1211">1211</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_positions</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1212" href="#t1212">1212</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">layer_norm_epsilon</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1213" href="#t1213">1213</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1214" href="#t1214">1214</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">activation_function</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1215" href="#t1215">1215</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1216" href="#t1216">1216</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1217" href="#t1217">1217</a></span><span class="t">            <span class="str">"trust_remote_code"</span><span class="op">:</span> <span class="str">"santacoder"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1218" href="#t1218">1218</a></span><span class="t">            <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">,</span>  <span class="com"># Only santacoder needs trust_remote_code</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1219" href="#t1219">1219</a></span><span class="t">            <span class="str">"scale_attn_by_inverse_layer_idx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">scale_attn_by_inverse_layer_idx</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1220" href="#t1220">1220</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"LN"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1221" href="#t1221">1221</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1222" href="#t1222">1222</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"LlamaForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1222&#x202F;&#x219B;&#x202F;1223</span><span class="annotate long">line 1222 didn't jump to line 1223</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1223" href="#t1223">1223</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1224" href="#t1224">1224</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1225" href="#t1225">1225</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1226" href="#t1226">1226</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1227" href="#t1227">1227</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">intermediate_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1228" href="#t1228">1228</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1229" href="#t1229">1229</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">max_position_embeddings</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1230" href="#t1230">1230</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rms_norm_eps</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1231" href="#t1231">1231</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1232" href="#t1232">1232</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_act</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1233" href="#t1233">1233</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1234" href="#t1234">1234</a></span><span class="t">                <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_key_value_heads</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1235" href="#t1235">1235</a></span><span class="t">                <span class="key">if</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_key_value_heads</span> <span class="op">!=</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1236" href="#t1236">1236</a></span><span class="t">                <span class="key">else</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1237" href="#t1237">1237</a></span><span class="t">            <span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1238" href="#t1238">1238</a></span><span class="t">            <span class="com"># This is done because the current implementation of GQA will use Grouped-Query Attention if</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1239" href="#t1239">1239</a></span><span class="t">            <span class="com"># n_key_value_heads is not None, but hf_config.num_key_value_heads is sometimes specified as</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1240" href="#t1240">1240</a></span><span class="t">            <span class="com"># the same as hf_config.num_attention_heads, in which case GQA should not be used.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1241" href="#t1241">1241</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1242" href="#t1242">1242</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1243" href="#t1243">1243</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1244" href="#t1244">1244</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1245" href="#t1245">1245</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1246" href="#t1246">1246</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1247" href="#t1247">1247</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1248" href="#t1248">1248</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"QWenLMHeadModel"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1248&#x202F;&#x219B;&#x202F;1249</span><span class="annotate long">line 1248 didn't jump to line 1249</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1249" href="#t1249">1249</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1250" href="#t1250">1250</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1251" href="#t1251">1251</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1252" href="#t1252">1252</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1253" href="#t1253">1253</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">intermediate_size</span> <span class="op">//</span> <span class="num">2</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1254" href="#t1254">1254</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1255" href="#t1255">1255</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>  <span class="com"># Capped bc the actual ctx length is 30k and the attn mask would be too big</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1256" href="#t1256">1256</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">layer_norm_epsilon</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1257" href="#t1257">1257</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1258" href="#t1258">1258</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1259" href="#t1259">1259</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">scale_attn_weights</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1260" href="#t1260">1260</a></span><span class="t">            <span class="str">"initializer_range"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">initializer_range</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1261" href="#t1261">1261</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1262" href="#t1262">1262</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1263" href="#t1263">1263</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">kv_channels</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1264" href="#t1264">1264</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1265" href="#t1265">1265</a></span><span class="t">            <span class="str">"tokenizer_prepends_bos"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1266" href="#t1266">1266</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1267" href="#t1267">1267</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1268" href="#t1268">1268</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1269" href="#t1269">1269</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"PhiForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1269&#x202F;&#x219B;&#x202F;1271</span><span class="annotate long">line 1269 didn't jump to line 1271</span></span></p>
-    <p class="pln"><span class="n"><a id="t1270" href="#t1270">1270</a></span><span class="t">        <span class="com"># Architecture for microsoft/phi models</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1271" href="#t1271">1271</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1272" href="#t1272">1272</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1273" href="#t1273">1273</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1274" href="#t1274">1274</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1275" href="#t1275">1275</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">intermediate_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1276" href="#t1276">1276</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1277" href="#t1277">1277</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">max_position_embeddings</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1278" href="#t1278">1278</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">layer_norm_eps</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1279" href="#t1279">1279</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1280" href="#t1280">1280</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_act</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1281" href="#t1281">1281</a></span><span class="t">            <span class="str">"initializer_range"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">initializer_range</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1282" href="#t1282">1282</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"LN"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1283" href="#t1283">1283</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1284" href="#t1284">1284</a></span><span class="t">            <span class="str">"trust_remote_code"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1285" href="#t1285">1285</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rope_theta</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1286" href="#t1286">1286</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1287" href="#t1287">1287</a></span><span class="t">            <span class="str">"parallel_attn_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1288" href="#t1288">1288</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1289" href="#t1289">1289</a></span><span class="t">        <span class="nam">partial_rotary_factor</span> <span class="op">=</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">partial_rotary_factor</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1290" href="#t1290">1290</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"rotary_dim"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">round</span><span class="op">(</span><span class="nam">partial_rotary_factor</span> <span class="op">*</span> <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"d_head"</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1291" href="#t1291">1291</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"Phi3ForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1291&#x202F;&#x219B;&#x202F;1293</span><span class="annotate long">line 1291 didn't jump to line 1293</span></span></p>
-    <p class="pln"><span class="n"><a id="t1292" href="#t1292">1292</a></span><span class="t">        <span class="com"># Architecture for microsoft/phi3 models</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1293" href="#t1293">1293</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1294" href="#t1294">1294</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1295" href="#t1295">1295</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1296" href="#t1296">1296</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1297" href="#t1297">1297</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">intermediate_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1298" href="#t1298">1298</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1299" href="#t1299">1299</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">max_position_embeddings</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1300" href="#t1300">1300</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rms_norm_eps</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1301" href="#t1301">1301</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1302" href="#t1302">1302</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_act</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1303" href="#t1303">1303</a></span><span class="t">            <span class="str">"initializer_range"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">initializer_range</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1304" href="#t1304">1304</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1305" href="#t1305">1305</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1306" href="#t1306">1306</a></span><span class="t">            <span class="str">"trust_remote_code"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1307" href="#t1307">1307</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rope_theta</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1308" href="#t1308">1308</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1309" href="#t1309">1309</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1310" href="#t1310">1310</a></span><span class="t">            <span class="str">"parallel_attn_mlp"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1311" href="#t1311">1311</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1312" href="#t1312">1312</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1313" href="#t1313">1313</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1314" href="#t1314">1314</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"google/gemma-2b"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1314&#x202F;&#x219B;&#x202F;1316</span><span class="annotate long">line 1314 didn't jump to line 1316</span></span></p>
-    <p class="pln"><span class="n"><a id="t1315" href="#t1315">1315</a></span><span class="t">        <span class="com"># Architecture for Gemma 2b and Gemma 2b Instruct models</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1316" href="#t1316">1316</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1317" href="#t1317">1317</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1318" href="#t1318">1318</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1319" href="#t1319">1319</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1320" href="#t1320">1320</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">16384</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1321" href="#t1321">1321</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">18</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1322" href="#t1322">1322</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1323" href="#t1323">1323</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-06</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1324" href="#t1324">1324</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">256000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1325" href="#t1325">1325</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"gelu_new"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1326" href="#t1326">1326</a></span><span class="t">            <span class="str">"initializer_range"</span><span class="op">:</span> <span class="num">0.02</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1327" href="#t1327">1327</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1328" href="#t1328">1328</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">10000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1329" href="#t1329">1329</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1330" href="#t1330">1330</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1331" href="#t1331">1331</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1332" href="#t1332">1332</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">1</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1333" href="#t1333">1333</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1334" href="#t1334">1334</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1335" href="#t1335">1335</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1336" href="#t1336">1336</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"google/gemma-7b"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1336&#x202F;&#x219B;&#x202F;1338</span><span class="annotate long">line 1336 didn't jump to line 1338</span></span></p>
-    <p class="pln"><span class="n"><a id="t1337" href="#t1337">1337</a></span><span class="t">        <span class="com"># Architecture for Gemma 7b and Gemma 7b Instruct models</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1338" href="#t1338">1338</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1339" href="#t1339">1339</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">3072</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1340" href="#t1340">1340</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1341" href="#t1341">1341</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">16</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1342" href="#t1342">1342</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">24576</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1343" href="#t1343">1343</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">28</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1344" href="#t1344">1344</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1345" href="#t1345">1345</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-06</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1346" href="#t1346">1346</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">256000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1347" href="#t1347">1347</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"gelu_new"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1348" href="#t1348">1348</a></span><span class="t">            <span class="str">"initializer_range"</span><span class="op">:</span> <span class="num">0.02</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1349" href="#t1349">1349</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1350" href="#t1350">1350</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">10000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1351" href="#t1351">1351</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1352" href="#t1352">1352</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1353" href="#t1353">1353</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1354" href="#t1354">1354</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">16</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1355" href="#t1355">1355</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1356" href="#t1356">1356</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1357" href="#t1357">1357</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1358" href="#t1358">1358</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"google/gemma-2-2b"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1358&#x202F;&#x219B;&#x202F;1360</span><span class="annotate long">line 1358 didn't jump to line 1360</span></span></p>
-    <p class="pln"><span class="n"><a id="t1359" href="#t1359">1359</a></span><span class="t">        <span class="com"># Architecture for Gemma-2 2b and Gemma-2 2b Instruct models</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1360" href="#t1360">1360</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1361" href="#t1361">1361</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">2304</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1362" href="#t1362">1362</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1363" href="#t1363">1363</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1364" href="#t1364">1364</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">9216</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1365" href="#t1365">1365</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">26</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1366" href="#t1366">1366</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1367" href="#t1367">1367</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-06</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1368" href="#t1368">1368</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">256000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1369" href="#t1369">1369</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"gelu_pytorch_tanh"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1370" href="#t1370">1370</a></span><span class="t">            <span class="str">"initializer_range"</span><span class="op">:</span> <span class="num">0.02</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1371" href="#t1371">1371</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1372" href="#t1372">1372</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">10000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1373" href="#t1373">1373</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1374" href="#t1374">1374</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1375" href="#t1375">1375</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">4</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1376" href="#t1376">1376</a></span><span class="t">            <span class="str">"window_size"</span><span class="op">:</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1377" href="#t1377">1377</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1378" href="#t1378">1378</a></span><span class="t">            <span class="str">"attn_types"</span><span class="op">:</span> <span class="op">[</span><span class="str">"global"</span><span class="op">,</span> <span class="str">"local"</span><span class="op">]</span> <span class="op">*</span> <span class="num">21</span><span class="op">,</span>  <span class="com"># Alternate global and local attn</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1379" href="#t1379">1379</a></span><span class="t">            <span class="str">"attn_scores_soft_cap"</span><span class="op">:</span> <span class="num">50.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1380" href="#t1380">1380</a></span><span class="t">            <span class="str">"output_logits_soft_cap"</span><span class="op">:</span> <span class="num">30.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1381" href="#t1381">1381</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1382" href="#t1382">1382</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1383" href="#t1383">1383</a></span><span class="t">            <span class="str">"use_normalization_before_and_after"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1266" href="#t1266">1266</a></span><span class="t">            <span class="str">"trust_remote_code"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1267" href="#t1267">1267</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1268" href="#t1268">1268</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1269" href="#t1269">1269</a></span><span class="t">            <span class="str">"default_prepend_bos"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1270" href="#t1270">1270</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1271" href="#t1271">1271</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"Qwen2ForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1272" href="#t1272">1272</a></span><span class="t">        <span class="com"># Note that Qwen1.5 models have architecture type Qwen2ForCausalLM.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1273" href="#t1273">1273</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1274" href="#t1274">1274</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1275" href="#t1275">1275</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1276" href="#t1276">1276</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1277" href="#t1277">1277</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_key_value_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1278" href="#t1278">1278</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">intermediate_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1279" href="#t1279">1279</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1280" href="#t1280">1280</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>  <span class="com"># Capped bc the actual ctx length is 30k and the attn mask would be too big</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1281" href="#t1281">1281</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rms_norm_eps</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1282" href="#t1282">1282</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1283" href="#t1283">1283</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_act</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1284" href="#t1284">1284</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1285" href="#t1285">1285</a></span><span class="t">            <span class="str">"initializer_range"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">initializer_range</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1286" href="#t1286">1286</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1287" href="#t1287">1287</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1288" href="#t1288">1288</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="nam">int</span><span class="op">(</span><span class="nam">hf_config</span><span class="op">.</span><span class="nam">rope_theta</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1289" href="#t1289">1289</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1290" href="#t1290">1290</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1291" href="#t1291">1291</a></span><span class="t">            <span class="str">"tokenizer_prepends_bos"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1292" href="#t1292">1292</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1293" href="#t1293">1293</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1294" href="#t1294">1294</a></span><span class="t">            <span class="str">"default_prepend_bos"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1295" href="#t1295">1295</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1296" href="#t1296">1296</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"PhiForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1296&#x202F;&#x219B;&#x202F;1298</span><span class="annotate long">line 1296 didn't jump to line 1298</span></span></p>
+    <p class="pln"><span class="n"><a id="t1297" href="#t1297">1297</a></span><span class="t">        <span class="com"># Architecture for microsoft/phi models</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1298" href="#t1298">1298</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1299" href="#t1299">1299</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1300" href="#t1300">1300</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1301" href="#t1301">1301</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1302" href="#t1302">1302</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">intermediate_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1303" href="#t1303">1303</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1304" href="#t1304">1304</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">max_position_embeddings</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1305" href="#t1305">1305</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">layer_norm_eps</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1306" href="#t1306">1306</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1307" href="#t1307">1307</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_act</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1308" href="#t1308">1308</a></span><span class="t">            <span class="str">"initializer_range"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">initializer_range</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1309" href="#t1309">1309</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"LN"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1310" href="#t1310">1310</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1311" href="#t1311">1311</a></span><span class="t">            <span class="str">"trust_remote_code"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1312" href="#t1312">1312</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rope_theta</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1313" href="#t1313">1313</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1314" href="#t1314">1314</a></span><span class="t">            <span class="str">"parallel_attn_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1315" href="#t1315">1315</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1316" href="#t1316">1316</a></span><span class="t">        <span class="nam">partial_rotary_factor</span> <span class="op">=</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">partial_rotary_factor</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1317" href="#t1317">1317</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"rotary_dim"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">round</span><span class="op">(</span><span class="nam">partial_rotary_factor</span> <span class="op">*</span> <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"d_head"</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1318" href="#t1318">1318</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"Phi3ForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1318&#x202F;&#x219B;&#x202F;1320</span><span class="annotate long">line 1318 didn't jump to line 1320</span></span></p>
+    <p class="pln"><span class="n"><a id="t1319" href="#t1319">1319</a></span><span class="t">        <span class="com"># Architecture for microsoft/phi3 models</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1320" href="#t1320">1320</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1321" href="#t1321">1321</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1322" href="#t1322">1322</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1323" href="#t1323">1323</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1324" href="#t1324">1324</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">intermediate_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1325" href="#t1325">1325</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1326" href="#t1326">1326</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">max_position_embeddings</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1327" href="#t1327">1327</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rms_norm_eps</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1328" href="#t1328">1328</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1329" href="#t1329">1329</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_act</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1330" href="#t1330">1330</a></span><span class="t">            <span class="str">"initializer_range"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">initializer_range</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1331" href="#t1331">1331</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1332" href="#t1332">1332</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1333" href="#t1333">1333</a></span><span class="t">            <span class="str">"trust_remote_code"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1334" href="#t1334">1334</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rope_theta</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1335" href="#t1335">1335</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1336" href="#t1336">1336</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1337" href="#t1337">1337</a></span><span class="t">            <span class="str">"parallel_attn_mlp"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1338" href="#t1338">1338</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1339" href="#t1339">1339</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1340" href="#t1340">1340</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1341" href="#t1341">1341</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"google/gemma-2b"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1341&#x202F;&#x219B;&#x202F;1343</span><span class="annotate long">line 1341 didn't jump to line 1343</span></span></p>
+    <p class="pln"><span class="n"><a id="t1342" href="#t1342">1342</a></span><span class="t">        <span class="com"># Architecture for Gemma 2b and Gemma 2b Instruct models</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1343" href="#t1343">1343</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1344" href="#t1344">1344</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1345" href="#t1345">1345</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1346" href="#t1346">1346</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1347" href="#t1347">1347</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">16384</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1348" href="#t1348">1348</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">18</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1349" href="#t1349">1349</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1350" href="#t1350">1350</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-06</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1351" href="#t1351">1351</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">256000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1352" href="#t1352">1352</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"gelu_new"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1353" href="#t1353">1353</a></span><span class="t">            <span class="str">"initializer_range"</span><span class="op">:</span> <span class="num">0.02</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1354" href="#t1354">1354</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1355" href="#t1355">1355</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">10000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1356" href="#t1356">1356</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1357" href="#t1357">1357</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1358" href="#t1358">1358</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1359" href="#t1359">1359</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">1</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1360" href="#t1360">1360</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1361" href="#t1361">1361</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1362" href="#t1362">1362</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1363" href="#t1363">1363</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"google/gemma-7b"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1363&#x202F;&#x219B;&#x202F;1365</span><span class="annotate long">line 1363 didn't jump to line 1365</span></span></p>
+    <p class="pln"><span class="n"><a id="t1364" href="#t1364">1364</a></span><span class="t">        <span class="com"># Architecture for Gemma 7b and Gemma 7b Instruct models</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1365" href="#t1365">1365</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1366" href="#t1366">1366</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">3072</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1367" href="#t1367">1367</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1368" href="#t1368">1368</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">16</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1369" href="#t1369">1369</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">24576</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1370" href="#t1370">1370</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">28</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1371" href="#t1371">1371</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1372" href="#t1372">1372</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-06</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1373" href="#t1373">1373</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">256000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1374" href="#t1374">1374</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"gelu_new"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1375" href="#t1375">1375</a></span><span class="t">            <span class="str">"initializer_range"</span><span class="op">:</span> <span class="num">0.02</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1376" href="#t1376">1376</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1377" href="#t1377">1377</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">10000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1378" href="#t1378">1378</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1379" href="#t1379">1379</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1380" href="#t1380">1380</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1381" href="#t1381">1381</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">16</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1382" href="#t1382">1382</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1383" href="#t1383">1383</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1384" href="#t1384">1384</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1385" href="#t1385">1385</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"google/gemma-2-9b"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1385&#x202F;&#x219B;&#x202F;1387</span><span class="annotate long">line 1385 didn't jump to line 1387</span></span></p>
-    <p class="pln"><span class="n"><a id="t1386" href="#t1386">1386</a></span><span class="t">        <span class="com"># Architecture for Gemma-2 9b and Gemma-2 9b Instruct models</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1385" href="#t1385">1385</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"google/gemma-2-2b"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1385&#x202F;&#x219B;&#x202F;1387</span><span class="annotate long">line 1385 didn't jump to line 1387</span></span></p>
+    <p class="pln"><span class="n"><a id="t1386" href="#t1386">1386</a></span><span class="t">        <span class="com"># Architecture for Gemma-2 2b and Gemma-2 2b Instruct models</span>&nbsp;</span><span class="r"></span></p>
     <p class="mis show_mis"><span class="n"><a id="t1387" href="#t1387">1387</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1388" href="#t1388">1388</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">3584</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1388" href="#t1388">1388</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">2304</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1389" href="#t1389">1389</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1390" href="#t1390">1390</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">16</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1391" href="#t1391">1391</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">14336</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1392" href="#t1392">1392</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">42</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1390" href="#t1390">1390</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1391" href="#t1391">1391</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">9216</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1392" href="#t1392">1392</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">26</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1393" href="#t1393">1393</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1394" href="#t1394">1394</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-06</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1395" href="#t1395">1395</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">256000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
@@ -1483,7 +1483,7 @@ <h2>
     <p class="pln"><span class="n"><a id="t1399" href="#t1399">1399</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">10000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1400" href="#t1400">1400</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1401" href="#t1401">1401</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1402" href="#t1402">1402</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1402" href="#t1402">1402</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">4</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1403" href="#t1403">1403</a></span><span class="t">            <span class="str">"window_size"</span><span class="op">:</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1404" href="#t1404">1404</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1405" href="#t1405">1405</a></span><span class="t">            <span class="str">"attn_types"</span><span class="op">:</span> <span class="op">[</span><span class="str">"global"</span><span class="op">,</span> <span class="str">"local"</span><span class="op">]</span> <span class="op">*</span> <span class="num">21</span><span class="op">,</span>  <span class="com"># Alternate global and local attn</span>&nbsp;</span><span class="r"></span></p>
@@ -1493,14 +1493,14 @@ <h2>
     <p class="pln"><span class="n"><a id="t1409" href="#t1409">1409</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1410" href="#t1410">1410</a></span><span class="t">            <span class="str">"use_normalization_before_and_after"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1411" href="#t1411">1411</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1412" href="#t1412">1412</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"google/gemma-2-27b"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1412&#x202F;&#x219B;&#x202F;1414</span><span class="annotate long">line 1412 didn't jump to line 1414</span></span></p>
-    <p class="pln"><span class="n"><a id="t1413" href="#t1413">1413</a></span><span class="t">        <span class="com"># Architecture for Gemma-2 27b and Gemma-2 27b Instruct models</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1412" href="#t1412">1412</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"google/gemma-2-9b"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1412&#x202F;&#x219B;&#x202F;1414</span><span class="annotate long">line 1412 didn't jump to line 1414</span></span></p>
+    <p class="pln"><span class="n"><a id="t1413" href="#t1413">1413</a></span><span class="t">        <span class="com"># Architecture for Gemma-2 9b and Gemma-2 9b Instruct models</span>&nbsp;</span><span class="r"></span></p>
     <p class="mis show_mis"><span class="n"><a id="t1414" href="#t1414">1414</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1415" href="#t1415">1415</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">4608</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1416" href="#t1416">1416</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1417" href="#t1417">1417</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1418" href="#t1418">1418</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">36864</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1419" href="#t1419">1419</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">46</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1415" href="#t1415">1415</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">3584</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1416" href="#t1416">1416</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1417" href="#t1417">1417</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">16</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1418" href="#t1418">1418</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">14336</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1419" href="#t1419">1419</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">42</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1420" href="#t1420">1420</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1421" href="#t1421">1421</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-06</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1422" href="#t1422">1422</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">256000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
@@ -1510,508 +1510,535 @@ <h2>
     <p class="pln"><span class="n"><a id="t1426" href="#t1426">1426</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">10000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1427" href="#t1427">1427</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1428" href="#t1428">1428</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1429" href="#t1429">1429</a></span><span class="t">            <span class="str">"attn_scale"</span><span class="op">:</span> <span class="num">12.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1430" href="#t1430">1430</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">16</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1431" href="#t1431">1431</a></span><span class="t">            <span class="str">"window_size"</span><span class="op">:</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1432" href="#t1432">1432</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1433" href="#t1433">1433</a></span><span class="t">            <span class="str">"attn_types"</span><span class="op">:</span> <span class="op">[</span><span class="str">"global"</span><span class="op">,</span> <span class="str">"local"</span><span class="op">]</span> <span class="op">*</span> <span class="num">23</span><span class="op">,</span>  <span class="com"># Alternate global and local attn</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1434" href="#t1434">1434</a></span><span class="t">            <span class="str">"attn_scores_soft_cap"</span><span class="op">:</span> <span class="num">50.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1435" href="#t1435">1435</a></span><span class="t">            <span class="str">"output_logits_soft_cap"</span><span class="op">:</span> <span class="num">30.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1436" href="#t1436">1436</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1437" href="#t1437">1437</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1438" href="#t1438">1438</a></span><span class="t">            <span class="str">"use_normalization_before_and_after"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1439" href="#t1439">1439</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1440" href="#t1440">1440</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"T5ForConditionalGeneration"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1440&#x202F;&#x219B;&#x202F;1460</span><span class="annotate long">line 1440 didn't jump to line 1460, because the condition on line 1440 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t1441" href="#t1441">1441</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1442" href="#t1442">1442</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">d_model</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1443" href="#t1443">1443</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">d_kv</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1444" href="#t1444">1444</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1445" href="#t1445">1445</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">d_ff</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1446" href="#t1446">1446</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1447" href="#t1447">1447</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1448" href="#t1448">1448</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">max_length</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1449" href="#t1449">1449</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">layer_norm_epsilon</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1450" href="#t1450">1450</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">feed_forward_proj</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1451" href="#t1451">1451</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"relative_positional_bias"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1452" href="#t1452">1452</a></span><span class="t">            <span class="str">"relative_attention_max_distance"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">relative_attention_max_distance</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1453" href="#t1453">1453</a></span><span class="t">            <span class="str">"relative_attention_num_buckets"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">relative_attention_num_buckets</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1454" href="#t1454">1454</a></span><span class="t">            <span class="str">"decoder_start_token_id"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">decoder_start_token_id</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1455" href="#t1455">1455</a></span><span class="t">            <span class="str">"attention_dir"</span><span class="op">:</span> <span class="str">"bidirectional"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1456" href="#t1456">1456</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1457" href="#t1457">1457</a></span><span class="t">            <span class="str">"tie_word_embeddings"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">tie_word_embeddings</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1458" href="#t1458">1458</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1459" href="#t1459">1459</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1460" href="#t1460">1460</a></span><span class="t">        <span class="key">raise</span> <span class="nam">NotImplementedError</span><span class="op">(</span><span class="str">f"{architecture} is not currently supported."</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1461" href="#t1461">1461</a></span><span class="t">    <span class="com"># All of these models use LayerNorm</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1462" href="#t1462">1462</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"original_architecture"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">architecture</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1463" href="#t1463">1463</a></span><span class="t">    <span class="com"># The name such that AutoTokenizer.from_pretrained works</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1464" href="#t1464">1464</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"tokenizer_name"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">official_model_name</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1465" href="#t1465">1465</a></span><span class="t">    <span class="key">if</span> <span class="nam">kwargs</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"trust_remote_code"</span><span class="op">,</span> <span class="key">False</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1465&#x202F;&#x219B;&#x202F;1466</span><span class="annotate long">line 1465 didn't jump to line 1466, because the condition on line 1465 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1466" href="#t1466">1466</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"trust_remote_code"</span><span class="op">]</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1467" href="#t1467">1467</a></span><span class="t">    <span class="key">return</span> <span class="nam">cfg_dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1468" href="#t1468">1468</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1469" href="#t1469">1469</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1470" href="#t1470">1470</a></span><span class="t"><span class="key">def</span> <span class="nam">convert_neel_model_config</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1471" href="#t1471">1471</a></span><span class="t">    <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1472" href="#t1472">1472</a></span><span class="t"><span class="str">    Loads the config for a model trained by me (NeelNanda), converted to a dictionary</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1473" href="#t1473">1473</a></span><span class="t"><span class="str">    in the HookedTransformerConfig format.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1474" href="#t1474">1474</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1475" href="#t1475">1475</a></span><span class="t"><span class="str">    AutoConfig is not supported, because these models are in the HookedTransformer format, so we directly download and load the json.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1476" href="#t1476">1476</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1477" href="#t1477">1477</a></span><span class="t">    <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">get_official_model_name</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1478" href="#t1478">1478</a></span><span class="t">    <span class="nam">cfg_json</span><span class="op">:</span> <span class="nam">dict</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">download_file_from_hf</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">,</span> <span class="str">"config.json"</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1479" href="#t1479">1479</a></span><span class="t">    <span class="nam">cfg_arch</span> <span class="op">=</span> <span class="nam">cfg_json</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1480" href="#t1480">1480</a></span><span class="t">        <span class="str">"architecture"</span><span class="op">,</span> <span class="str">"neel"</span> <span class="key">if</span> <span class="str">"_old"</span> <span class="key">not</span> <span class="key">in</span> <span class="nam">official_model_name</span> <span class="key">else</span> <span class="str">"neel-solu-old"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1481" href="#t1481">1481</a></span><span class="t">    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1482" href="#t1482">1482</a></span><span class="t">    <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1483" href="#t1483">1483</a></span><span class="t">        <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1484" href="#t1484">1484</a></span><span class="t">        <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"n_layers"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1485" href="#t1485">1485</a></span><span class="t">        <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"d_mlp"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1486" href="#t1486">1486</a></span><span class="t">        <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"d_head"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1487" href="#t1487">1487</a></span><span class="t">        <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"n_heads"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1488" href="#t1488">1488</a></span><span class="t">        <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"n_ctx"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1489" href="#t1489">1489</a></span><span class="t">        <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"d_vocab"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1490" href="#t1490">1490</a></span><span class="t">        <span class="str">"tokenizer_name"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"tokenizer_name"</span><span class="op">,</span> <span class="key">None</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1491" href="#t1491">1491</a></span><span class="t">        <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"act_fn"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1492" href="#t1492">1492</a></span><span class="t">        <span class="str">"attn_only"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"attn_only"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1493" href="#t1493">1493</a></span><span class="t">        <span class="str">"final_rms"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"final_rms"</span><span class="op">,</span> <span class="key">False</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1494" href="#t1494">1494</a></span><span class="t">        <span class="str">"original_architecture"</span><span class="op">:</span> <span class="nam">cfg_arch</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1495" href="#t1495">1495</a></span><span class="t">    <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1496" href="#t1496">1496</a></span><span class="t">    <span class="key">if</span> <span class="str">"normalization"</span> <span class="key">in</span> <span class="nam">cfg_json</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1497" href="#t1497">1497</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"normalization_type"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"normalization"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1498" href="#t1498">1498</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1499" href="#t1499">1499</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"normalization_type"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"normalization_type"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1500" href="#t1500">1500</a></span><span class="t">    <span class="key">if</span> <span class="str">"shortformer_pos"</span> <span class="key">in</span> <span class="nam">cfg_json</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1501" href="#t1501">1501</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"positional_embedding_type"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1502" href="#t1502">1502</a></span><span class="t">            <span class="str">"shortformer"</span> <span class="key">if</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"shortformer_pos"</span><span class="op">]</span> <span class="key">else</span> <span class="str">"standard"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1503" href="#t1503">1503</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1504" href="#t1504">1504</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1505" href="#t1505">1505</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"positional_embedding_type"</span><span class="op">]</span> <span class="op">=</span> <span class="str">"standard"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1506" href="#t1506">1506</a></span><span class="t">    <span class="key">return</span> <span class="nam">cfg_dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1507" href="#t1507">1507</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1508" href="#t1508">1508</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1509" href="#t1509">1509</a></span><span class="t"><span class="key">def</span> <span class="nam">get_pretrained_model_config</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1510" href="#t1510">1510</a></span><span class="t">    <span class="nam">model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1511" href="#t1511">1511</a></span><span class="t">    <span class="nam">hf_cfg</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">dict</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1512" href="#t1512">1512</a></span><span class="t">    <span class="nam">checkpoint_index</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1513" href="#t1513">1513</a></span><span class="t">    <span class="nam">checkpoint_value</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1514" href="#t1514">1514</a></span><span class="t">    <span class="nam">fold_ln</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1515" href="#t1515">1515</a></span><span class="t">    <span class="nam">device</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">device</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1516" href="#t1516">1516</a></span><span class="t">    <span class="nam">n_devices</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">1</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1517" href="#t1517">1517</a></span><span class="t">    <span class="nam">default_prepend_bos</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">bool</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1518" href="#t1518">1518</a></span><span class="t">    <span class="nam">dtype</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">dtype</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1519" href="#t1519">1519</a></span><span class="t">    <span class="nam">first_n_layers</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1520" href="#t1520">1520</a></span><span class="t">    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1521" href="#t1521">1521</a></span><span class="t"><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1522" href="#t1522">1522</a></span><span class="t">    <span class="str">"""Returns the pretrained model config as an HookedTransformerConfig object.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1523" href="#t1523">1523</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1524" href="#t1524">1524</a></span><span class="t"><span class="str">    There are two types of pretrained models: HuggingFace models (where</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1525" href="#t1525">1525</a></span><span class="t"><span class="str">    AutoModel and AutoConfig work), and models trained by me (NeelNanda) which</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1526" href="#t1526">1526</a></span><span class="t"><span class="str">    aren't as integrated with HuggingFace infrastructure.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1527" href="#t1527">1527</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1528" href="#t1528">1528</a></span><span class="t"><span class="str">    Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1529" href="#t1529">1529</a></span><span class="t"><span class="str">        model_name: The name of the model. This can be either the official</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1530" href="#t1530">1530</a></span><span class="t"><span class="str">            HuggingFace model name, or the name of a model trained by me</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1531" href="#t1531">1531</a></span><span class="t"><span class="str">            (NeelNanda).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1532" href="#t1532">1532</a></span><span class="t"><span class="str">        hf_cfg (dict, optional): Config of a loaded pretrained HF model,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1533" href="#t1533">1533</a></span><span class="t"><span class="str">            converted to a dictionary.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1534" href="#t1534">1534</a></span><span class="t"><span class="str">        checkpoint_index (int, optional): If loading from a</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1535" href="#t1535">1535</a></span><span class="t"><span class="str">            checkpoint, the index of the checkpoint to load. Defaults to None.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1536" href="#t1536">1536</a></span><span class="t"><span class="str">        checkpoint_value (int, optional): If loading from a checkpoint, the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1537" href="#t1537">1537</a></span><span class="t"><span class="str">        value of</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1538" href="#t1538">1538</a></span><span class="t"><span class="str">            the checkpoint to load, ie the step or token number (each model has</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1539" href="#t1539">1539</a></span><span class="t"><span class="str">            checkpoints labelled with exactly one of these). Defaults to None.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1540" href="#t1540">1540</a></span><span class="t"><span class="str">        fold_ln (bool, optional): Whether to fold the layer norm into the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1541" href="#t1541">1541</a></span><span class="t"><span class="str">            subsequent linear layers (see HookedTransformer.fold_layer_norm for</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1542" href="#t1542">1542</a></span><span class="t"><span class="str">            details). Defaults to False.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1543" href="#t1543">1543</a></span><span class="t"><span class="str">        device (str, optional): The device to load the model onto. By</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1544" href="#t1544">1544</a></span><span class="t"><span class="str">            default will load to CUDA if available, else CPU.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1545" href="#t1545">1545</a></span><span class="t"><span class="str">        n_devices (int, optional): The number of devices to split the model across. Defaults to 1.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1546" href="#t1546">1546</a></span><span class="t"><span class="str">        default_prepend_bos (bool, optional): Default behavior of whether to prepend the BOS token when the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1547" href="#t1547">1547</a></span><span class="t"><span class="str">            methods of HookedTransformer process input text to tokenize (only when input is a string).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1548" href="#t1548">1548</a></span><span class="t"><span class="str">            Resolution order for default_prepend_bos:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1549" href="#t1549">1549</a></span><span class="t"><span class="str">            1. If user passes value explicitly, use that value</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1550" href="#t1550">1550</a></span><span class="t"><span class="str">            2. Model-specific default from cfg_dict if it exists (e.g. for bloom models it's False)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1551" href="#t1551">1551</a></span><span class="t"><span class="str">            3. Global default (True)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1552" href="#t1552">1552</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1553" href="#t1553">1553</a></span><span class="t"><span class="str">            Even for models not explicitly trained with the BOS token, heads often use the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1554" href="#t1554">1554</a></span><span class="t"><span class="str">            first position as a resting position and accordingly lose information from the first token,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1555" href="#t1555">1555</a></span><span class="t"><span class="str">            so this empirically seems to give better results. Note that you can also locally override the default behavior</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1556" href="#t1556">1556</a></span><span class="t"><span class="str">            by passing in prepend_bos=True/False when you call a method that processes the input string.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1557" href="#t1557">1557</a></span><span class="t"><span class="str">        dtype (torch.dtype, optional): The dtype to load the TransformerLens model in.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1558" href="#t1558">1558</a></span><span class="t"><span class="str">        kwargs: Other optional arguments passed to HuggingFace's from_pretrained.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1559" href="#t1559">1559</a></span><span class="t"><span class="str">            Also given to other HuggingFace functions when compatible.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1560" href="#t1560">1560</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1561" href="#t1561">1561</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1562" href="#t1562">1562</a></span><span class="t">    <span class="key">if</span> <span class="nam">Path</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span><span class="op">.</span><span class="nam">exists</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1562&#x202F;&#x219B;&#x202F;1564</span><span class="annotate long">line 1562 didn't jump to line 1564, because the condition on line 1562 was never true</span></span></p>
-    <p class="pln"><span class="n"><a id="t1563" href="#t1563">1563</a></span><span class="t">        <span class="com"># If the model_name is a path, it's a local model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1564" href="#t1564">1564</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="nam">convert_hf_model_config</span><span class="op">(</span><span class="nam">model_name</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1565" href="#t1565">1565</a></span><span class="t">        <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">model_name</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1566" href="#t1566">1566</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1567" href="#t1567">1567</a></span><span class="t">        <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">get_official_model_name</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1568" href="#t1568">1568</a></span><span class="t">    <span class="key">if</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1569" href="#t1569">1569</a></span><span class="t">        <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"NeelNanda"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1570" href="#t1570">1570</a></span><span class="t">        <span class="key">or</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"ArthurConmy"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1571" href="#t1571">1571</a></span><span class="t">        <span class="key">or</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"Baidicoot"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1572" href="#t1572">1572</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1573" href="#t1573">1573</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="nam">convert_neel_model_config</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1574" href="#t1574">1574</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1575" href="#t1575">1575</a></span><span class="t">        <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="nam">NEED_REMOTE_CODE_MODELS</span><span class="op">)</span> <span class="key">and</span> <span class="key">not</span> <span class="nam">kwargs</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span>&nbsp;</span><span class="r"><span class="annotate short">1575&#x202F;&#x219B;&#x202F;1578</span><span class="annotate long">line 1575 didn't jump to line 1578, because the condition on line 1575 was never true</span></span></p>
-    <p class="pln"><span class="n"><a id="t1576" href="#t1576">1576</a></span><span class="t">            <span class="str">"trust_remote_code"</span><span class="op">,</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1577" href="#t1577">1577</a></span><span class="t">        <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1578" href="#t1578">1578</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1579" href="#t1579">1579</a></span><span class="t">                <span class="str">f"Loading model {official_model_name} requires setting trust_remote_code=True"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1580" href="#t1580">1580</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1581" href="#t1581">1581</a></span><span class="t">            <span class="nam">kwargs</span><span class="op">[</span><span class="str">"trust_remote_code"</span><span class="op">]</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1582" href="#t1582">1582</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="nam">convert_hf_model_config</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1583" href="#t1583">1583</a></span><span class="t">    <span class="com"># Processing common to both model types</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1584" href="#t1584">1584</a></span><span class="t">    <span class="com"># Remove any prefix, saying the organization who made a model.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1585" href="#t1585">1585</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"model_name"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">split</span><span class="op">(</span><span class="str">"/"</span><span class="op">)</span><span class="op">[</span><span class="op">-</span><span class="num">1</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1586" href="#t1586">1586</a></span><span class="t">    <span class="com"># Don't need to initialize weights, we're loading from pretrained</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1587" href="#t1587">1587</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"init_weights"</span><span class="op">]</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1588" href="#t1588">1588</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1589" href="#t1589">1589</a></span><span class="t">    <span class="key">if</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1590" href="#t1590">1590</a></span><span class="t">        <span class="str">"positional_embedding_type"</span> <span class="key">in</span> <span class="nam">cfg_dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1591" href="#t1591">1591</a></span><span class="t">        <span class="key">and</span> <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"positional_embedding_type"</span><span class="op">]</span> <span class="op">==</span> <span class="str">"shortformer"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1592" href="#t1592">1592</a></span><span class="t">        <span class="key">and</span> <span class="nam">fold_ln</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1593" href="#t1593">1593</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1594" href="#t1594">1594</a></span><span class="t">        <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1595" href="#t1595">1595</a></span><span class="t">            <span class="str">"You tried to specify fold_ln=True for a shortformer model, but this can't be done! Setting fold_ln=False instead."</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1596" href="#t1596">1596</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1597" href="#t1597">1597</a></span><span class="t">        <span class="nam">fold_ln</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1598" href="#t1598">1598</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1599" href="#t1599">1599</a></span><span class="t">    <span class="key">if</span> <span class="nam">device</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1600" href="#t1600">1600</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"device"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">device</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1601" href="#t1601">1601</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1602" href="#t1602">1602</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"dtype"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">dtype</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1603" href="#t1603">1603</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1604" href="#t1604">1604</a></span><span class="t">    <span class="key">if</span> <span class="nam">fold_ln</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1605" href="#t1605">1605</a></span><span class="t">        <span class="key">if</span> <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"normalization_type"</span><span class="op">]</span> <span class="key">in</span> <span class="op">[</span><span class="str">"LN"</span><span class="op">,</span> <span class="str">"LNPre"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1605&#x202F;&#x219B;&#x202F;1607</span><span class="annotate long">line 1605 didn't jump to line 1607, because the condition on line 1605 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t1606" href="#t1606">1606</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"normalization_type"</span><span class="op">]</span> <span class="op">=</span> <span class="str">"LNPre"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1607" href="#t1607">1607</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"normalization_type"</span><span class="op">]</span> <span class="key">in</span> <span class="op">[</span><span class="str">"RMS"</span><span class="op">,</span> <span class="str">"RMSPre"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1608" href="#t1608">1608</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"normalization_type"</span><span class="op">]</span> <span class="op">=</span> <span class="str">"RMSPre"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1609" href="#t1609">1609</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1610" href="#t1610">1610</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span><span class="str">"Cannot fold in layer norm, normalization_type is not LN."</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1611" href="#t1611">1611</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1612" href="#t1612">1612</a></span><span class="t">    <span class="key">if</span> <span class="nam">checkpoint_index</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span> <span class="key">or</span> <span class="nam">checkpoint_value</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1612&#x202F;&#x219B;&#x202F;1613</span><span class="annotate long">line 1612 didn't jump to line 1613, because the condition on line 1612 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1613" href="#t1613">1613</a></span><span class="t">        <span class="nam">checkpoint_labels</span><span class="op">,</span> <span class="nam">checkpoint_label_type</span> <span class="op">=</span> <span class="nam">get_checkpoint_labels</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1614" href="#t1614">1614</a></span><span class="t">            <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1615" href="#t1615">1615</a></span><span class="t">            <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1616" href="#t1616">1616</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1617" href="#t1617">1617</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"from_checkpoint"</span><span class="op">]</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1618" href="#t1618">1618</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"checkpoint_label_type"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">checkpoint_label_type</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1619" href="#t1619">1619</a></span><span class="t">        <span class="key">if</span> <span class="nam">checkpoint_index</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1620" href="#t1620">1620</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"checkpoint_index"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">checkpoint_index</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1621" href="#t1621">1621</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"checkpoint_value"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">checkpoint_labels</span><span class="op">[</span><span class="nam">checkpoint_index</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1622" href="#t1622">1622</a></span><span class="t">        <span class="key">elif</span> <span class="nam">checkpoint_value</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1623" href="#t1623">1623</a></span><span class="t">            <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1624" href="#t1624">1624</a></span><span class="t">                <span class="nam">checkpoint_value</span> <span class="key">in</span> <span class="nam">checkpoint_labels</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1625" href="#t1625">1625</a></span><span class="t">            <span class="op">)</span><span class="op">,</span> <span class="str">f"Checkpoint value {checkpoint_value} is not in list of available checkpoints"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1626" href="#t1626">1626</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"checkpoint_value"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">checkpoint_value</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1627" href="#t1627">1627</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"checkpoint_index"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">checkpoint_labels</span><span class="op">.</span><span class="nam">index</span><span class="op">(</span><span class="nam">checkpoint_value</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1628" href="#t1628">1628</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1629" href="#t1629">1629</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"from_checkpoint"</span><span class="op">]</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1429" href="#t1429">1429</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1430" href="#t1430">1430</a></span><span class="t">            <span class="str">"window_size"</span><span class="op">:</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1431" href="#t1431">1431</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1432" href="#t1432">1432</a></span><span class="t">            <span class="str">"attn_types"</span><span class="op">:</span> <span class="op">[</span><span class="str">"global"</span><span class="op">,</span> <span class="str">"local"</span><span class="op">]</span> <span class="op">*</span> <span class="num">21</span><span class="op">,</span>  <span class="com"># Alternate global and local attn</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1433" href="#t1433">1433</a></span><span class="t">            <span class="str">"attn_scores_soft_cap"</span><span class="op">:</span> <span class="num">50.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1434" href="#t1434">1434</a></span><span class="t">            <span class="str">"output_logits_soft_cap"</span><span class="op">:</span> <span class="num">30.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1435" href="#t1435">1435</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1436" href="#t1436">1436</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1437" href="#t1437">1437</a></span><span class="t">            <span class="str">"use_normalization_before_and_after"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1438" href="#t1438">1438</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1439" href="#t1439">1439</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"google/gemma-2-27b"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1439&#x202F;&#x219B;&#x202F;1441</span><span class="annotate long">line 1439 didn't jump to line 1441</span></span></p>
+    <p class="pln"><span class="n"><a id="t1440" href="#t1440">1440</a></span><span class="t">        <span class="com"># Architecture for Gemma-2 27b and Gemma-2 27b Instruct models</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1441" href="#t1441">1441</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1442" href="#t1442">1442</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">4608</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1443" href="#t1443">1443</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1444" href="#t1444">1444</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1445" href="#t1445">1445</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">36864</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1446" href="#t1446">1446</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">46</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1447" href="#t1447">1447</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1448" href="#t1448">1448</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-06</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1449" href="#t1449">1449</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">256000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1450" href="#t1450">1450</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"gelu_pytorch_tanh"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1451" href="#t1451">1451</a></span><span class="t">            <span class="str">"initializer_range"</span><span class="op">:</span> <span class="num">0.02</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1452" href="#t1452">1452</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1453" href="#t1453">1453</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">10000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1454" href="#t1454">1454</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1455" href="#t1455">1455</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1456" href="#t1456">1456</a></span><span class="t">            <span class="str">"attn_scale"</span><span class="op">:</span> <span class="num">12.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1457" href="#t1457">1457</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">16</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1458" href="#t1458">1458</a></span><span class="t">            <span class="str">"window_size"</span><span class="op">:</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1459" href="#t1459">1459</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1460" href="#t1460">1460</a></span><span class="t">            <span class="str">"attn_types"</span><span class="op">:</span> <span class="op">[</span><span class="str">"global"</span><span class="op">,</span> <span class="str">"local"</span><span class="op">]</span> <span class="op">*</span> <span class="num">23</span><span class="op">,</span>  <span class="com"># Alternate global and local attn</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1461" href="#t1461">1461</a></span><span class="t">            <span class="str">"attn_scores_soft_cap"</span><span class="op">:</span> <span class="num">50.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1462" href="#t1462">1462</a></span><span class="t">            <span class="str">"output_logits_soft_cap"</span><span class="op">:</span> <span class="num">30.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1463" href="#t1463">1463</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1464" href="#t1464">1464</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1465" href="#t1465">1465</a></span><span class="t">            <span class="str">"use_normalization_before_and_after"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1466" href="#t1466">1466</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1467" href="#t1467">1467</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"T5ForConditionalGeneration"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1467&#x202F;&#x219B;&#x202F;1487</span><span class="annotate long">line 1467 didn't jump to line 1487, because the condition on line 1467 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t1468" href="#t1468">1468</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1469" href="#t1469">1469</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">d_model</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1470" href="#t1470">1470</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">d_kv</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1471" href="#t1471">1471</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1472" href="#t1472">1472</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">d_ff</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1473" href="#t1473">1473</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1474" href="#t1474">1474</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1475" href="#t1475">1475</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">max_length</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1476" href="#t1476">1476</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">layer_norm_epsilon</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1477" href="#t1477">1477</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">feed_forward_proj</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1478" href="#t1478">1478</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"relative_positional_bias"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1479" href="#t1479">1479</a></span><span class="t">            <span class="str">"relative_attention_max_distance"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">relative_attention_max_distance</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1480" href="#t1480">1480</a></span><span class="t">            <span class="str">"relative_attention_num_buckets"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">relative_attention_num_buckets</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1481" href="#t1481">1481</a></span><span class="t">            <span class="str">"decoder_start_token_id"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">decoder_start_token_id</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1482" href="#t1482">1482</a></span><span class="t">            <span class="str">"attention_dir"</span><span class="op">:</span> <span class="str">"bidirectional"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1483" href="#t1483">1483</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1484" href="#t1484">1484</a></span><span class="t">            <span class="str">"tie_word_embeddings"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">tie_word_embeddings</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1485" href="#t1485">1485</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1486" href="#t1486">1486</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1487" href="#t1487">1487</a></span><span class="t">        <span class="key">raise</span> <span class="nam">NotImplementedError</span><span class="op">(</span><span class="str">f"{architecture} is not currently supported."</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1488" href="#t1488">1488</a></span><span class="t">    <span class="com"># All of these models use LayerNorm</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1489" href="#t1489">1489</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"original_architecture"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">architecture</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1490" href="#t1490">1490</a></span><span class="t">    <span class="com"># The name such that AutoTokenizer.from_pretrained works</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1491" href="#t1491">1491</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"tokenizer_name"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">official_model_name</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1492" href="#t1492">1492</a></span><span class="t">    <span class="key">if</span> <span class="nam">kwargs</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"trust_remote_code"</span><span class="op">,</span> <span class="key">False</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1492&#x202F;&#x219B;&#x202F;1493</span><span class="annotate long">line 1492 didn't jump to line 1493, because the condition on line 1492 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1493" href="#t1493">1493</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"trust_remote_code"</span><span class="op">]</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1494" href="#t1494">1494</a></span><span class="t">    <span class="key">return</span> <span class="nam">cfg_dict</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1495" href="#t1495">1495</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1496" href="#t1496">1496</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1497" href="#t1497">1497</a></span><span class="t"><span class="key">def</span> <span class="nam">convert_neel_model_config</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1498" href="#t1498">1498</a></span><span class="t">    <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1499" href="#t1499">1499</a></span><span class="t"><span class="str">    Loads the config for a model trained by me (NeelNanda), converted to a dictionary</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1500" href="#t1500">1500</a></span><span class="t"><span class="str">    in the HookedTransformerConfig format.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1501" href="#t1501">1501</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1502" href="#t1502">1502</a></span><span class="t"><span class="str">    AutoConfig is not supported, because these models are in the HookedTransformer format, so we directly download and load the json.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1503" href="#t1503">1503</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1504" href="#t1504">1504</a></span><span class="t">    <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">get_official_model_name</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1505" href="#t1505">1505</a></span><span class="t">    <span class="nam">cfg_json</span><span class="op">:</span> <span class="nam">dict</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">download_file_from_hf</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">,</span> <span class="str">"config.json"</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1506" href="#t1506">1506</a></span><span class="t">    <span class="nam">cfg_arch</span> <span class="op">=</span> <span class="nam">cfg_json</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1507" href="#t1507">1507</a></span><span class="t">        <span class="str">"architecture"</span><span class="op">,</span> <span class="str">"neel"</span> <span class="key">if</span> <span class="str">"_old"</span> <span class="key">not</span> <span class="key">in</span> <span class="nam">official_model_name</span> <span class="key">else</span> <span class="str">"neel-solu-old"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1508" href="#t1508">1508</a></span><span class="t">    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1509" href="#t1509">1509</a></span><span class="t">    <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1510" href="#t1510">1510</a></span><span class="t">        <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1511" href="#t1511">1511</a></span><span class="t">        <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"n_layers"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1512" href="#t1512">1512</a></span><span class="t">        <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"d_mlp"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1513" href="#t1513">1513</a></span><span class="t">        <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"d_head"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1514" href="#t1514">1514</a></span><span class="t">        <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"n_heads"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1515" href="#t1515">1515</a></span><span class="t">        <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"n_ctx"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1516" href="#t1516">1516</a></span><span class="t">        <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"d_vocab"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1517" href="#t1517">1517</a></span><span class="t">        <span class="str">"tokenizer_name"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"tokenizer_name"</span><span class="op">,</span> <span class="key">None</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1518" href="#t1518">1518</a></span><span class="t">        <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"act_fn"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1519" href="#t1519">1519</a></span><span class="t">        <span class="str">"attn_only"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"attn_only"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1520" href="#t1520">1520</a></span><span class="t">        <span class="str">"final_rms"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"final_rms"</span><span class="op">,</span> <span class="key">False</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1521" href="#t1521">1521</a></span><span class="t">        <span class="str">"original_architecture"</span><span class="op">:</span> <span class="nam">cfg_arch</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1522" href="#t1522">1522</a></span><span class="t">    <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1523" href="#t1523">1523</a></span><span class="t">    <span class="key">if</span> <span class="str">"normalization"</span> <span class="key">in</span> <span class="nam">cfg_json</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1524" href="#t1524">1524</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"normalization_type"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"normalization"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1525" href="#t1525">1525</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1526" href="#t1526">1526</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"normalization_type"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"normalization_type"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1527" href="#t1527">1527</a></span><span class="t">    <span class="key">if</span> <span class="str">"shortformer_pos"</span> <span class="key">in</span> <span class="nam">cfg_json</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1528" href="#t1528">1528</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"positional_embedding_type"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1529" href="#t1529">1529</a></span><span class="t">            <span class="str">"shortformer"</span> <span class="key">if</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"shortformer_pos"</span><span class="op">]</span> <span class="key">else</span> <span class="str">"standard"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1530" href="#t1530">1530</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1531" href="#t1531">1531</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1532" href="#t1532">1532</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"positional_embedding_type"</span><span class="op">]</span> <span class="op">=</span> <span class="str">"standard"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1533" href="#t1533">1533</a></span><span class="t">    <span class="key">return</span> <span class="nam">cfg_dict</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1534" href="#t1534">1534</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1535" href="#t1535">1535</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1536" href="#t1536">1536</a></span><span class="t"><span class="key">def</span> <span class="nam">get_pretrained_model_config</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1537" href="#t1537">1537</a></span><span class="t">    <span class="nam">model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1538" href="#t1538">1538</a></span><span class="t">    <span class="nam">hf_cfg</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">dict</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1539" href="#t1539">1539</a></span><span class="t">    <span class="nam">checkpoint_index</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1540" href="#t1540">1540</a></span><span class="t">    <span class="nam">checkpoint_value</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1541" href="#t1541">1541</a></span><span class="t">    <span class="nam">fold_ln</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1542" href="#t1542">1542</a></span><span class="t">    <span class="nam">device</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">device</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1543" href="#t1543">1543</a></span><span class="t">    <span class="nam">n_devices</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">1</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1544" href="#t1544">1544</a></span><span class="t">    <span class="nam">default_prepend_bos</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">bool</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1545" href="#t1545">1545</a></span><span class="t">    <span class="nam">dtype</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">dtype</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1546" href="#t1546">1546</a></span><span class="t">    <span class="nam">first_n_layers</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1547" href="#t1547">1547</a></span><span class="t">    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1548" href="#t1548">1548</a></span><span class="t"><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1549" href="#t1549">1549</a></span><span class="t">    <span class="str">"""Returns the pretrained model config as an HookedTransformerConfig object.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1550" href="#t1550">1550</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1551" href="#t1551">1551</a></span><span class="t"><span class="str">    There are two types of pretrained models: HuggingFace models (where</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1552" href="#t1552">1552</a></span><span class="t"><span class="str">    AutoModel and AutoConfig work), and models trained by me (NeelNanda) which</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1553" href="#t1553">1553</a></span><span class="t"><span class="str">    aren't as integrated with HuggingFace infrastructure.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1554" href="#t1554">1554</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1555" href="#t1555">1555</a></span><span class="t"><span class="str">    Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1556" href="#t1556">1556</a></span><span class="t"><span class="str">        model_name: The name of the model. This can be either the official</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1557" href="#t1557">1557</a></span><span class="t"><span class="str">            HuggingFace model name, or the name of a model trained by me</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1558" href="#t1558">1558</a></span><span class="t"><span class="str">            (NeelNanda).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1559" href="#t1559">1559</a></span><span class="t"><span class="str">        hf_cfg (dict, optional): Config of a loaded pretrained HF model,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1560" href="#t1560">1560</a></span><span class="t"><span class="str">            converted to a dictionary.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1561" href="#t1561">1561</a></span><span class="t"><span class="str">        checkpoint_index (int, optional): If loading from a</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1562" href="#t1562">1562</a></span><span class="t"><span class="str">            checkpoint, the index of the checkpoint to load. Defaults to None.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1563" href="#t1563">1563</a></span><span class="t"><span class="str">        checkpoint_value (int, optional): If loading from a checkpoint, the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1564" href="#t1564">1564</a></span><span class="t"><span class="str">        value of</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1565" href="#t1565">1565</a></span><span class="t"><span class="str">            the checkpoint to load, ie the step or token number (each model has</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1566" href="#t1566">1566</a></span><span class="t"><span class="str">            checkpoints labelled with exactly one of these). Defaults to None.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1567" href="#t1567">1567</a></span><span class="t"><span class="str">        fold_ln (bool, optional): Whether to fold the layer norm into the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1568" href="#t1568">1568</a></span><span class="t"><span class="str">            subsequent linear layers (see HookedTransformer.fold_layer_norm for</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1569" href="#t1569">1569</a></span><span class="t"><span class="str">            details). Defaults to False.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1570" href="#t1570">1570</a></span><span class="t"><span class="str">        device (str, optional): The device to load the model onto. By</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1571" href="#t1571">1571</a></span><span class="t"><span class="str">            default will load to CUDA if available, else CPU.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1572" href="#t1572">1572</a></span><span class="t"><span class="str">        n_devices (int, optional): The number of devices to split the model across. Defaults to 1.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1573" href="#t1573">1573</a></span><span class="t"><span class="str">        default_prepend_bos (bool, optional): Default behavior of whether to prepend the BOS token when the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1574" href="#t1574">1574</a></span><span class="t"><span class="str">            methods of HookedTransformer process input text to tokenize (only when input is a string).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1575" href="#t1575">1575</a></span><span class="t"><span class="str">            Resolution order for default_prepend_bos:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1576" href="#t1576">1576</a></span><span class="t"><span class="str">            1. If user passes value explicitly, use that value</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1577" href="#t1577">1577</a></span><span class="t"><span class="str">            2. Model-specific default from cfg_dict if it exists (e.g. for bloom models it's False)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1578" href="#t1578">1578</a></span><span class="t"><span class="str">            3. Global default (True)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1579" href="#t1579">1579</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1580" href="#t1580">1580</a></span><span class="t"><span class="str">            Even for models not explicitly trained with the BOS token, heads often use the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1581" href="#t1581">1581</a></span><span class="t"><span class="str">            first position as a resting position and accordingly lose information from the first token,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1582" href="#t1582">1582</a></span><span class="t"><span class="str">            so this empirically seems to give better results. Note that you can also locally override the default behavior</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1583" href="#t1583">1583</a></span><span class="t"><span class="str">            by passing in prepend_bos=True/False when you call a method that processes the input string.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1584" href="#t1584">1584</a></span><span class="t"><span class="str">        dtype (torch.dtype, optional): The dtype to load the TransformerLens model in.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1585" href="#t1585">1585</a></span><span class="t"><span class="str">        kwargs: Other optional arguments passed to HuggingFace's from_pretrained.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1586" href="#t1586">1586</a></span><span class="t"><span class="str">            Also given to other HuggingFace functions when compatible.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1587" href="#t1587">1587</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1588" href="#t1588">1588</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1589" href="#t1589">1589</a></span><span class="t">    <span class="key">if</span> <span class="nam">Path</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span><span class="op">.</span><span class="nam">exists</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1589&#x202F;&#x219B;&#x202F;1591</span><span class="annotate long">line 1589 didn't jump to line 1591, because the condition on line 1589 was never true</span></span></p>
+    <p class="pln"><span class="n"><a id="t1590" href="#t1590">1590</a></span><span class="t">        <span class="com"># If the model_name is a path, it's a local model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1591" href="#t1591">1591</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="nam">convert_hf_model_config</span><span class="op">(</span><span class="nam">model_name</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1592" href="#t1592">1592</a></span><span class="t">        <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">model_name</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1593" href="#t1593">1593</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1594" href="#t1594">1594</a></span><span class="t">        <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">get_official_model_name</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1595" href="#t1595">1595</a></span><span class="t">    <span class="key">if</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1596" href="#t1596">1596</a></span><span class="t">        <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"NeelNanda"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1597" href="#t1597">1597</a></span><span class="t">        <span class="key">or</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"ArthurConmy"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1598" href="#t1598">1598</a></span><span class="t">        <span class="key">or</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"Baidicoot"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1599" href="#t1599">1599</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1600" href="#t1600">1600</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="nam">convert_neel_model_config</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1601" href="#t1601">1601</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1602" href="#t1602">1602</a></span><span class="t">        <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="nam">NEED_REMOTE_CODE_MODELS</span><span class="op">)</span> <span class="key">and</span> <span class="key">not</span> <span class="nam">kwargs</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span>&nbsp;</span><span class="r"><span class="annotate short">1602&#x202F;&#x219B;&#x202F;1605</span><span class="annotate long">line 1602 didn't jump to line 1605, because the condition on line 1602 was never true</span></span></p>
+    <p class="pln"><span class="n"><a id="t1603" href="#t1603">1603</a></span><span class="t">            <span class="str">"trust_remote_code"</span><span class="op">,</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1604" href="#t1604">1604</a></span><span class="t">        <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1605" href="#t1605">1605</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1606" href="#t1606">1606</a></span><span class="t">                <span class="str">f"Loading model {official_model_name} requires setting trust_remote_code=True"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1607" href="#t1607">1607</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1608" href="#t1608">1608</a></span><span class="t">            <span class="nam">kwargs</span><span class="op">[</span><span class="str">"trust_remote_code"</span><span class="op">]</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1609" href="#t1609">1609</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="nam">convert_hf_model_config</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1610" href="#t1610">1610</a></span><span class="t">    <span class="com"># Processing common to both model types</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1611" href="#t1611">1611</a></span><span class="t">    <span class="com"># Remove any prefix, saying the organization who made a model.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1612" href="#t1612">1612</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"model_name"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">split</span><span class="op">(</span><span class="str">"/"</span><span class="op">)</span><span class="op">[</span><span class="op">-</span><span class="num">1</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1613" href="#t1613">1613</a></span><span class="t">    <span class="com"># Don't need to initialize weights, we're loading from pretrained</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1614" href="#t1614">1614</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"init_weights"</span><span class="op">]</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1615" href="#t1615">1615</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1616" href="#t1616">1616</a></span><span class="t">    <span class="key">if</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1617" href="#t1617">1617</a></span><span class="t">        <span class="str">"positional_embedding_type"</span> <span class="key">in</span> <span class="nam">cfg_dict</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1618" href="#t1618">1618</a></span><span class="t">        <span class="key">and</span> <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"positional_embedding_type"</span><span class="op">]</span> <span class="op">==</span> <span class="str">"shortformer"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1619" href="#t1619">1619</a></span><span class="t">        <span class="key">and</span> <span class="nam">fold_ln</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1620" href="#t1620">1620</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1621" href="#t1621">1621</a></span><span class="t">        <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1622" href="#t1622">1622</a></span><span class="t">            <span class="str">"You tried to specify fold_ln=True for a shortformer model, but this can't be done! Setting fold_ln=False instead."</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1623" href="#t1623">1623</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1624" href="#t1624">1624</a></span><span class="t">        <span class="nam">fold_ln</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1625" href="#t1625">1625</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1626" href="#t1626">1626</a></span><span class="t">    <span class="key">if</span> <span class="nam">device</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1627" href="#t1627">1627</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"device"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">device</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1628" href="#t1628">1628</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1629" href="#t1629">1629</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"dtype"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">dtype</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1630" href="#t1630">1630</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1631" href="#t1631">1631</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"device"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">device</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1632" href="#t1632">1632</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"n_devices"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">n_devices</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1633" href="#t1633">1633</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1634" href="#t1634">1634</a></span><span class="t">    <span class="key">if</span> <span class="nam">default_prepend_bos</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1635" href="#t1635">1635</a></span><span class="t">        <span class="com"># User explicitly set prepend_bos behavior, override config/default value</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1636" href="#t1636">1636</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"default_prepend_bos"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">default_prepend_bos</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1637" href="#t1637">1637</a></span><span class="t">    <span class="key">elif</span> <span class="str">"default_prepend_bos"</span> <span class="key">not</span> <span class="key">in</span> <span class="nam">cfg_dict</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1638" href="#t1638">1638</a></span><span class="t">        <span class="com"># No config value or user override, set default value (True)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1639" href="#t1639">1639</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"default_prepend_bos"</span><span class="op">]</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1640" href="#t1640">1640</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1641" href="#t1641">1641</a></span><span class="t">    <span class="key">if</span> <span class="nam">hf_cfg</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1642" href="#t1642">1642</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"load_in_4bit"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">hf_cfg</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"quantization_config"</span><span class="op">,</span> <span class="op">{</span><span class="op">}</span><span class="op">)</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"load_in_4bit"</span><span class="op">,</span> <span class="key">False</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1643" href="#t1643">1643</a></span><span class="t">    <span class="key">if</span> <span class="nam">first_n_layers</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1643&#x202F;&#x219B;&#x202F;1644</span><span class="annotate long">line 1643 didn't jump to line 1644, because the condition on line 1643 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1644" href="#t1644">1644</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"n_layers"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">first_n_layers</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1645" href="#t1645">1645</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1646" href="#t1646">1646</a></span><span class="t">    <span class="nam">cfg</span> <span class="op">=</span> <span class="nam">HookedTransformerConfig</span><span class="op">.</span><span class="nam">from_dict</span><span class="op">(</span><span class="nam">cfg_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1647" href="#t1647">1647</a></span><span class="t">    <span class="key">return</span> <span class="nam">cfg</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1648" href="#t1648">1648</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1649" href="#t1649">1649</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1650" href="#t1650">1650</a></span><span class="t"><span class="key">def</span> <span class="nam">get_num_params_of_pretrained</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1651" href="#t1651">1651</a></span><span class="t">    <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1652" href="#t1652">1652</a></span><span class="t"><span class="str">    Returns the number of parameters of a pretrained model, used to filter to only run code for sufficiently small models.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1653" href="#t1653">1653</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1654" href="#t1654">1654</a></span><span class="t">    <span class="nam">cfg</span> <span class="op">=</span> <span class="nam">get_pretrained_model_config</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1655" href="#t1655">1655</a></span><span class="t">    <span class="key">return</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">n_params</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1656" href="#t1656">1656</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1631" href="#t1631">1631</a></span><span class="t">    <span class="key">if</span> <span class="nam">fold_ln</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1632" href="#t1632">1632</a></span><span class="t">        <span class="key">if</span> <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"normalization_type"</span><span class="op">]</span> <span class="key">in</span> <span class="op">[</span><span class="str">"LN"</span><span class="op">,</span> <span class="str">"LNPre"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1633" href="#t1633">1633</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"normalization_type"</span><span class="op">]</span> <span class="op">=</span> <span class="str">"LNPre"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1634" href="#t1634">1634</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"normalization_type"</span><span class="op">]</span> <span class="key">in</span> <span class="op">[</span><span class="str">"RMS"</span><span class="op">,</span> <span class="str">"RMSPre"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1634&#x202F;&#x219B;&#x202F;1637</span><span class="annotate long">line 1634 didn't jump to line 1637, because the condition on line 1634 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t1635" href="#t1635">1635</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"normalization_type"</span><span class="op">]</span> <span class="op">=</span> <span class="str">"RMSPre"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1636" href="#t1636">1636</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1637" href="#t1637">1637</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span><span class="str">"Cannot fold in layer norm, normalization_type is not LN."</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1638" href="#t1638">1638</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1639" href="#t1639">1639</a></span><span class="t">    <span class="key">if</span> <span class="nam">checkpoint_index</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span> <span class="key">or</span> <span class="nam">checkpoint_value</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1639&#x202F;&#x219B;&#x202F;1640</span><span class="annotate long">line 1639 didn't jump to line 1640, because the condition on line 1639 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1640" href="#t1640">1640</a></span><span class="t">        <span class="nam">checkpoint_labels</span><span class="op">,</span> <span class="nam">checkpoint_label_type</span> <span class="op">=</span> <span class="nam">get_checkpoint_labels</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1641" href="#t1641">1641</a></span><span class="t">            <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1642" href="#t1642">1642</a></span><span class="t">            <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1643" href="#t1643">1643</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1644" href="#t1644">1644</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"from_checkpoint"</span><span class="op">]</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1645" href="#t1645">1645</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"checkpoint_label_type"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">checkpoint_label_type</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1646" href="#t1646">1646</a></span><span class="t">        <span class="key">if</span> <span class="nam">checkpoint_index</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1647" href="#t1647">1647</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"checkpoint_index"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">checkpoint_index</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1648" href="#t1648">1648</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"checkpoint_value"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">checkpoint_labels</span><span class="op">[</span><span class="nam">checkpoint_index</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1649" href="#t1649">1649</a></span><span class="t">        <span class="key">elif</span> <span class="nam">checkpoint_value</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1650" href="#t1650">1650</a></span><span class="t">            <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1651" href="#t1651">1651</a></span><span class="t">                <span class="nam">checkpoint_value</span> <span class="key">in</span> <span class="nam">checkpoint_labels</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1652" href="#t1652">1652</a></span><span class="t">            <span class="op">)</span><span class="op">,</span> <span class="str">f"Checkpoint value {checkpoint_value} is not in list of available checkpoints"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1653" href="#t1653">1653</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"checkpoint_value"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">checkpoint_value</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1654" href="#t1654">1654</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"checkpoint_index"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">checkpoint_labels</span><span class="op">.</span><span class="nam">index</span><span class="op">(</span><span class="nam">checkpoint_value</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1655" href="#t1655">1655</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1656" href="#t1656">1656</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"from_checkpoint"</span><span class="op">]</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1657" href="#t1657">1657</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1658" href="#t1658">1658</a></span><span class="t"><span class="com"># %% Load checkpointed model state dicts</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1659" href="#t1659">1659</a></span><span class="t"><span class="com"># The steps for which there are checkpoints in the stanford crfm models</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1660" href="#t1660">1660</a></span><span class="t"><span class="nam">STANFORD_CRFM_CHECKPOINTS</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1661" href="#t1661">1661</a></span><span class="t">    <span class="nam">list</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="num">100</span><span class="op">,</span> <span class="num">10</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1662" href="#t1662">1662</a></span><span class="t">    <span class="op">+</span> <span class="nam">list</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="num">100</span><span class="op">,</span> <span class="num">2000</span><span class="op">,</span> <span class="num">50</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1663" href="#t1663">1663</a></span><span class="t">    <span class="op">+</span> <span class="nam">list</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="num">2000</span><span class="op">,</span> <span class="num">20000</span><span class="op">,</span> <span class="num">100</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1664" href="#t1664">1664</a></span><span class="t">    <span class="op">+</span> <span class="nam">list</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="num">20000</span><span class="op">,</span> <span class="num">400000</span> <span class="op">+</span> <span class="num">1</span><span class="op">,</span> <span class="num">1000</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1665" href="#t1665">1665</a></span><span class="t"><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1666" href="#t1666">1666</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1667" href="#t1667">1667</a></span><span class="t"><span class="com"># Linearly spaced checkpoints for Pythia models, taken every 1000 steps.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1668" href="#t1668">1668</a></span><span class="t"><span class="com"># Batch size 2,097,152 tokens, so checkpoints every 2.1B tokens</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1669" href="#t1669">1669</a></span><span class="t"><span class="nam">PYTHIA_CHECKPOINTS</span> <span class="op">=</span> <span class="op">[</span><span class="num">0</span><span class="op">,</span> <span class="num">1</span><span class="op">,</span> <span class="num">2</span><span class="op">,</span> <span class="num">4</span><span class="op">,</span> <span class="num">8</span><span class="op">,</span> <span class="num">16</span><span class="op">,</span> <span class="num">32</span><span class="op">,</span> <span class="num">64</span><span class="op">,</span> <span class="num">128</span><span class="op">,</span> <span class="num">256</span><span class="op">,</span> <span class="num">512</span><span class="op">]</span> <span class="op">+</span> <span class="nam">list</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1670" href="#t1670">1670</a></span><span class="t">    <span class="nam">range</span><span class="op">(</span><span class="num">1000</span><span class="op">,</span> <span class="num">143000</span> <span class="op">+</span> <span class="num">1</span><span class="op">,</span> <span class="num">1000</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1671" href="#t1671">1671</a></span><span class="t"><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1672" href="#t1672">1672</a></span><span class="t"><span class="com"># Pythia V1 has log-spaced early checkpoints (see line above), but V0 doesn't</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1673" href="#t1673">1673</a></span><span class="t"><span class="nam">PYTHIA_V0_CHECKPOINTS</span> <span class="op">=</span> <span class="nam">list</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="num">1000</span><span class="op">,</span> <span class="num">143000</span> <span class="op">+</span> <span class="num">1</span><span class="op">,</span> <span class="num">1000</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1674" href="#t1674">1674</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1658" href="#t1658">1658</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"device"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">device</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1659" href="#t1659">1659</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"n_devices"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">n_devices</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1660" href="#t1660">1660</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1661" href="#t1661">1661</a></span><span class="t">    <span class="key">if</span> <span class="nam">default_prepend_bos</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1662" href="#t1662">1662</a></span><span class="t">        <span class="com"># User explicitly set prepend_bos behavior, override config/default value</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1663" href="#t1663">1663</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"default_prepend_bos"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">default_prepend_bos</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1664" href="#t1664">1664</a></span><span class="t">    <span class="key">elif</span> <span class="str">"default_prepend_bos"</span> <span class="key">not</span> <span class="key">in</span> <span class="nam">cfg_dict</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1665" href="#t1665">1665</a></span><span class="t">        <span class="com"># No config value or user override, set default value (True)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1666" href="#t1666">1666</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"default_prepend_bos"</span><span class="op">]</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1667" href="#t1667">1667</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1668" href="#t1668">1668</a></span><span class="t">    <span class="key">if</span> <span class="nam">hf_cfg</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1669" href="#t1669">1669</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"load_in_4bit"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">hf_cfg</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"quantization_config"</span><span class="op">,</span> <span class="op">{</span><span class="op">}</span><span class="op">)</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"load_in_4bit"</span><span class="op">,</span> <span class="key">False</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1670" href="#t1670">1670</a></span><span class="t">    <span class="key">if</span> <span class="nam">first_n_layers</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1670&#x202F;&#x219B;&#x202F;1671</span><span class="annotate long">line 1670 didn't jump to line 1671, because the condition on line 1670 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1671" href="#t1671">1671</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"n_layers"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">first_n_layers</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1672" href="#t1672">1672</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1673" href="#t1673">1673</a></span><span class="t">    <span class="nam">cfg</span> <span class="op">=</span> <span class="nam">HookedTransformerConfig</span><span class="op">.</span><span class="nam">from_dict</span><span class="op">(</span><span class="nam">cfg_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1674" href="#t1674">1674</a></span><span class="t">    <span class="key">return</span> <span class="nam">cfg</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1675" href="#t1675">1675</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1676" href="#t1676">1676</a></span><span class="t"><span class="key">def</span> <span class="nam">get_checkpoint_labels</span><span class="op">(</span><span class="nam">model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1677" href="#t1677">1677</a></span><span class="t">    <span class="str">"""Returns the checkpoint labels for a given model, and the label_type</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1678" href="#t1678">1678</a></span><span class="t"><span class="str">    (step or token). Raises an error for models that are not checkpointed."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1679" href="#t1679">1679</a></span><span class="t">    <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">get_official_model_name</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1680" href="#t1680">1680</a></span><span class="t">    <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"stanford-crfm/"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1681" href="#t1681">1681</a></span><span class="t">        <span class="key">return</span> <span class="nam">STANFORD_CRFM_CHECKPOINTS</span><span class="op">,</span> <span class="str">"step"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1682" href="#t1682">1682</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"EleutherAI/pythia"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1683" href="#t1683">1683</a></span><span class="t">        <span class="key">if</span> <span class="str">"v0"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1684" href="#t1684">1684</a></span><span class="t">            <span class="key">return</span> <span class="nam">PYTHIA_V0_CHECKPOINTS</span><span class="op">,</span> <span class="str">"step"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1685" href="#t1685">1685</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1686" href="#t1686">1686</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1687" href="#t1687">1687</a></span><span class="t">                <span class="str">"Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models."</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1688" href="#t1688">1688</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1689" href="#t1689">1689</a></span><span class="t">            <span class="key">return</span> <span class="nam">PYTHIA_CHECKPOINTS</span><span class="op">,</span> <span class="str">"step"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1690" href="#t1690">1690</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"NeelNanda/"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1691" href="#t1691">1691</a></span><span class="t">        <span class="nam">api</span> <span class="op">=</span> <span class="nam">HfApi</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1692" href="#t1692">1692</a></span><span class="t">        <span class="nam">files_list</span> <span class="op">=</span> <span class="nam">api</span><span class="op">.</span><span class="nam">list_repo_files</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1693" href="#t1693">1693</a></span><span class="t">            <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1694" href="#t1694">1694</a></span><span class="t">            <span class="op">**</span><span class="nam">utils</span><span class="op">.</span><span class="nam">select_compatible_kwargs</span><span class="op">(</span><span class="nam">kwargs</span><span class="op">,</span> <span class="nam">api</span><span class="op">.</span><span class="nam">list_repo_files</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1695" href="#t1695">1695</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1696" href="#t1696">1696</a></span><span class="t">        <span class="nam">labels</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1697" href="#t1697">1697</a></span><span class="t">        <span class="key">for</span> <span class="nam">file_name</span> <span class="key">in</span> <span class="nam">files_list</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1698" href="#t1698">1698</a></span><span class="t">            <span class="nam">match</span> <span class="op">=</span> <span class="nam">re</span><span class="op">.</span><span class="nam">match</span><span class="op">(</span><span class="str">r"checkpoints/.*_(\d*)\.pth"</span><span class="op">,</span> <span class="nam">file_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1699" href="#t1699">1699</a></span><span class="t">            <span class="key">if</span> <span class="nam">match</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1700" href="#t1700">1700</a></span><span class="t">                <span class="nam">labels</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">int</span><span class="op">(</span><span class="nam">match</span><span class="op">.</span><span class="nam">group</span><span class="op">(</span><span class="num">1</span><span class="op">)</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1701" href="#t1701">1701</a></span><span class="t">        <span class="key">if</span> <span class="nam">labels</span><span class="op">[</span><span class="op">-</span><span class="num">1</span><span class="op">]</span> <span class="op">></span> <span class="num">1e9</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1702" href="#t1702">1702</a></span><span class="t">            <span class="nam">label_type</span> <span class="op">=</span> <span class="str">"token"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1703" href="#t1703">1703</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1704" href="#t1704">1704</a></span><span class="t">            <span class="nam">label_type</span> <span class="op">=</span> <span class="str">"step"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1705" href="#t1705">1705</a></span><span class="t">        <span class="key">return</span> <span class="nam">labels</span><span class="op">,</span> <span class="nam">label_type</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1706" href="#t1706">1706</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1707" href="#t1707">1707</a></span><span class="t">        <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">f"Model {official_model_name} is not checkpointed."</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1708" href="#t1708">1708</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1709" href="#t1709">1709</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1710" href="#t1710">1710</a></span><span class="t"><span class="com"># %% Loading state dicts</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1711" href="#t1711">1711</a></span><span class="t"><span class="key">def</span> <span class="nam">get_pretrained_state_dict</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1712" href="#t1712">1712</a></span><span class="t">    <span class="nam">official_model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1713" href="#t1713">1713</a></span><span class="t">    <span class="nam">cfg</span><span class="op">:</span> <span class="nam">HookedTransformerConfig</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1714" href="#t1714">1714</a></span><span class="t">    <span class="nam">hf_model</span><span class="op">=</span><span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1715" href="#t1715">1715</a></span><span class="t">    <span class="nam">dtype</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">dtype</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1716" href="#t1716">1716</a></span><span class="t">    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1717" href="#t1717">1717</a></span><span class="t"><span class="op">)</span> <span class="op">-></span> <span class="nam">Dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1718" href="#t1718">1718</a></span><span class="t">    <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1719" href="#t1719">1719</a></span><span class="t"><span class="str">    Loads in the model weights for a pretrained model, and processes them to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1720" href="#t1720">1720</a></span><span class="t"><span class="str">    have the HookedTransformer parameter names and shapes. Supports checkpointed</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1721" href="#t1721">1721</a></span><span class="t"><span class="str">    models (and expects the checkpoint info to be stored in the config object)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1722" href="#t1722">1722</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1723" href="#t1723">1723</a></span><span class="t"><span class="str">    hf_model: Optionally, a HuggingFace model object. If provided, we will use</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1724" href="#t1724">1724</a></span><span class="t"><span class="str">        these weights rather than reloading the model.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1725" href="#t1725">1725</a></span><span class="t"><span class="str">    dtype: The dtype to load the HuggingFace model in.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1726" href="#t1726">1726</a></span><span class="t"><span class="str">    kwargs: Other optional arguments passed to HuggingFace's from_pretrained.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1727" href="#t1727">1727</a></span><span class="t"><span class="str">        Also given to other HuggingFace functions when compatible.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1728" href="#t1728">1728</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1729" href="#t1729">1729</a></span><span class="t">    <span class="key">if</span> <span class="str">"torch_dtype"</span> <span class="key">in</span> <span class="nam">kwargs</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1729&#x202F;&#x219B;&#x202F;1730</span><span class="annotate long">line 1729 didn't jump to line 1730, because the condition on line 1729 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1730" href="#t1730">1730</a></span><span class="t">        <span class="nam">dtype</span> <span class="op">=</span> <span class="nam">kwargs</span><span class="op">[</span><span class="str">"torch_dtype"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1731" href="#t1731">1731</a></span><span class="t">        <span class="key">del</span> <span class="nam">kwargs</span><span class="op">[</span><span class="str">"torch_dtype"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1732" href="#t1732">1732</a></span><span class="t">    <span class="key">if</span> <span class="nam">Path</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">)</span><span class="op">.</span><span class="nam">exists</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1732&#x202F;&#x219B;&#x202F;1733</span><span class="annotate long">line 1732 didn't jump to line 1733, because the condition on line 1732 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1733" href="#t1733">1733</a></span><span class="t">        <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">str</span><span class="op">(</span><span class="nam">Path</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">)</span><span class="op">.</span><span class="nam">resolve</span><span class="op">(</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1734" href="#t1734">1734</a></span><span class="t">        <span class="nam">logging</span><span class="op">.</span><span class="nam">info</span><span class="op">(</span><span class="str">f"Loading model from local path {official_model_name}"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1735" href="#t1735">1735</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1736" href="#t1736">1736</a></span><span class="t">        <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">get_official_model_name</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1737" href="#t1737">1737</a></span><span class="t">    <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="nam">NEED_REMOTE_CODE_MODELS</span><span class="op">)</span> <span class="key">and</span> <span class="key">not</span> <span class="nam">kwargs</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span>&nbsp;</span><span class="r"><span class="annotate short">1737&#x202F;&#x219B;&#x202F;1740</span><span class="annotate long">line 1737 didn't jump to line 1740, because the condition on line 1737 was never true</span></span></p>
-    <p class="pln"><span class="n"><a id="t1738" href="#t1738">1738</a></span><span class="t">        <span class="str">"trust_remote_code"</span><span class="op">,</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1739" href="#t1739">1739</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1740" href="#t1740">1740</a></span><span class="t">        <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1741" href="#t1741">1741</a></span><span class="t">            <span class="str">f"Loading model {official_model_name} state dict requires setting trust_remote_code=True"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1742" href="#t1742">1742</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1743" href="#t1743">1743</a></span><span class="t">        <span class="nam">kwargs</span><span class="op">[</span><span class="str">"trust_remote_code"</span><span class="op">]</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1744" href="#t1744">1744</a></span><span class="t">    <span class="key">if</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1745" href="#t1745">1745</a></span><span class="t">        <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"NeelNanda"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1746" href="#t1746">1746</a></span><span class="t">        <span class="key">or</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"ArthurConmy"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1747" href="#t1747">1747</a></span><span class="t">        <span class="key">or</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"Baidicoot"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1748" href="#t1748">1748</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1749" href="#t1749">1749</a></span><span class="t">        <span class="nam">api</span> <span class="op">=</span> <span class="nam">HfApi</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1750" href="#t1750">1750</a></span><span class="t">        <span class="nam">repo_files</span> <span class="op">=</span> <span class="nam">api</span><span class="op">.</span><span class="nam">list_repo_files</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1751" href="#t1751">1751</a></span><span class="t">            <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1752" href="#t1752">1752</a></span><span class="t">            <span class="op">**</span><span class="nam">utils</span><span class="op">.</span><span class="nam">select_compatible_kwargs</span><span class="op">(</span><span class="nam">kwargs</span><span class="op">,</span> <span class="nam">api</span><span class="op">.</span><span class="nam">list_repo_files</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1753" href="#t1753">1753</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1754" href="#t1754">1754</a></span><span class="t">        <span class="key">if</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">from_checkpoint</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1754&#x202F;&#x219B;&#x202F;1755</span><span class="annotate long">line 1754 didn't jump to line 1755, because the condition on line 1754 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1755" href="#t1755">1755</a></span><span class="t">            <span class="nam">file_name</span> <span class="op">=</span> <span class="nam">list</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1756" href="#t1756">1756</a></span><span class="t">                <span class="nam">filter</span><span class="op">(</span><span class="key">lambda</span> <span class="nam">x</span><span class="op">:</span> <span class="nam">x</span><span class="op">.</span><span class="nam">endswith</span><span class="op">(</span><span class="str">f"{cfg.checkpoint_value}.pth"</span><span class="op">)</span><span class="op">,</span> <span class="nam">repo_files</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1757" href="#t1757">1757</a></span><span class="t">            <span class="op">)</span><span class="op">[</span><span class="num">0</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1758" href="#t1758">1758</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1759" href="#t1759">1759</a></span><span class="t">            <span class="nam">file_name</span> <span class="op">=</span> <span class="nam">list</span><span class="op">(</span><span class="nam">filter</span><span class="op">(</span><span class="key">lambda</span> <span class="nam">x</span><span class="op">:</span> <span class="nam">x</span><span class="op">.</span><span class="nam">endswith</span><span class="op">(</span><span class="str">"final.pth"</span><span class="op">)</span><span class="op">,</span> <span class="nam">repo_files</span><span class="op">)</span><span class="op">)</span><span class="op">[</span><span class="num">0</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1760" href="#t1760">1760</a></span><span class="t">        <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">download_file_from_hf</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">,</span> <span class="nam">file_name</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1761" href="#t1761">1761</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1762" href="#t1762">1762</a></span><span class="t">        <span class="com"># Convert to dtype</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1763" href="#t1763">1763</a></span><span class="t">        <span class="nam">state_dict</span> <span class="op">=</span> <span class="op">{</span><span class="nam">k</span><span class="op">:</span> <span class="nam">v</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">dtype</span><span class="op">)</span> <span class="key">for</span> <span class="nam">k</span><span class="op">,</span> <span class="nam">v</span> <span class="key">in</span> <span class="nam">state_dict</span><span class="op">.</span><span class="nam">items</span><span class="op">(</span><span class="op">)</span><span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1764" href="#t1764">1764</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1765" href="#t1765">1765</a></span><span class="t">        <span class="key">if</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"neel-solu-old"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1766" href="#t1766">1766</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_neel_solu_old_weights</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1767" href="#t1767">1767</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"mingpt"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1768" href="#t1768">1768</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_mingpt_weights</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1769" href="#t1769">1769</a></span><span class="t">        <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1770" href="#t1770">1770</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1771" href="#t1771">1771</a></span><span class="t">        <span class="key">if</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">from_checkpoint</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1771&#x202F;&#x219B;&#x202F;1772</span><span class="annotate long">line 1771 didn't jump to line 1772, because the condition on line 1771 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1772" href="#t1772">1772</a></span><span class="t">            <span class="nam">huggingface_token</span> <span class="op">=</span> <span class="nam">os</span><span class="op">.</span><span class="nam">environ</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"HF_TOKEN"</span><span class="op">,</span> <span class="key">None</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1773" href="#t1773">1773</a></span><span class="t">            <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"stanford-crfm"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1774" href="#t1774">1774</a></span><span class="t">                <span class="nam">hf_model</span> <span class="op">=</span> <span class="nam">AutoModelForCausalLM</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1775" href="#t1775">1775</a></span><span class="t">                    <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1776" href="#t1776">1776</a></span><span class="t">                    <span class="nam">revision</span><span class="op">=</span><span class="str">f"checkpoint-{cfg.checkpoint_value}"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1777" href="#t1777">1777</a></span><span class="t">                    <span class="nam">torch_dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1778" href="#t1778">1778</a></span><span class="t">                    <span class="nam">token</span><span class="op">=</span><span class="nam">huggingface_token</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1779" href="#t1779">1779</a></span><span class="t">                    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1780" href="#t1780">1780</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1781" href="#t1781">1781</a></span><span class="t">            <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"EleutherAI/pythia"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1782" href="#t1782">1782</a></span><span class="t">                <span class="nam">hf_model</span> <span class="op">=</span> <span class="nam">AutoModelForCausalLM</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1783" href="#t1783">1783</a></span><span class="t">                    <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1784" href="#t1784">1784</a></span><span class="t">                    <span class="nam">revision</span><span class="op">=</span><span class="str">f"step{cfg.checkpoint_value}"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1785" href="#t1785">1785</a></span><span class="t">                    <span class="nam">torch_dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1786" href="#t1786">1786</a></span><span class="t">                    <span class="nam">token</span><span class="op">=</span><span class="nam">huggingface_token</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1787" href="#t1787">1787</a></span><span class="t">                    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1788" href="#t1788">1788</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1789" href="#t1789">1789</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1790" href="#t1790">1790</a></span><span class="t">                <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">f"Checkpoints for model {official_model_name} are not supported"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1791" href="#t1791">1791</a></span><span class="t">        <span class="key">elif</span> <span class="nam">hf_model</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1791&#x202F;&#x219B;&#x202F;1819</span><span class="annotate long">line 1791 didn't jump to line 1819, because the condition on line 1791 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t1792" href="#t1792">1792</a></span><span class="t">            <span class="nam">huggingface_token</span> <span class="op">=</span> <span class="nam">os</span><span class="op">.</span><span class="nam">environ</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"HF_TOKEN"</span><span class="op">,</span> <span class="key">None</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1793" href="#t1793">1793</a></span><span class="t">            <span class="key">if</span> <span class="nam">official_model_name</span> <span class="key">in</span> <span class="nam">NON_HF_HOSTED_MODEL_NAMES</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1793&#x202F;&#x219B;&#x202F;1794</span><span class="annotate long">line 1793 didn't jump to line 1794, because the condition on line 1793 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1794" href="#t1794">1794</a></span><span class="t">                <span class="key">raise</span> <span class="nam">NotImplementedError</span><span class="op">(</span><span class="str">"Model not hosted on HuggingFace, must pass in hf_model"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1795" href="#t1795">1795</a></span><span class="t">            <span class="key">elif</span> <span class="str">"bert"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1796" href="#t1796">1796</a></span><span class="t">                <span class="nam">hf_model</span> <span class="op">=</span> <span class="nam">BertForPreTraining</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1797" href="#t1797">1797</a></span><span class="t">                    <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1798" href="#t1798">1798</a></span><span class="t">                    <span class="nam">torch_dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1799" href="#t1799">1799</a></span><span class="t">                    <span class="nam">token</span><span class="op">=</span><span class="nam">huggingface_token</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1800" href="#t1800">1800</a></span><span class="t">                    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1801" href="#t1801">1801</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1802" href="#t1802">1802</a></span><span class="t">            <span class="key">elif</span> <span class="str">"t5"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1803" href="#t1803">1803</a></span><span class="t">                <span class="nam">hf_model</span> <span class="op">=</span> <span class="nam">T5ForConditionalGeneration</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1804" href="#t1804">1804</a></span><span class="t">                    <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1805" href="#t1805">1805</a></span><span class="t">                    <span class="nam">torch_dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1806" href="#t1806">1806</a></span><span class="t">                    <span class="nam">token</span><span class="op">=</span><span class="nam">huggingface_token</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1807" href="#t1807">1807</a></span><span class="t">                    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1808" href="#t1808">1808</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1809" href="#t1809">1809</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1810" href="#t1810">1810</a></span><span class="t">                <span class="nam">hf_model</span> <span class="op">=</span> <span class="nam">AutoModelForCausalLM</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1811" href="#t1811">1811</a></span><span class="t">                    <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1676" href="#t1676">1676</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1677" href="#t1677">1677</a></span><span class="t"><span class="key">def</span> <span class="nam">get_num_params_of_pretrained</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1678" href="#t1678">1678</a></span><span class="t">    <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1679" href="#t1679">1679</a></span><span class="t"><span class="str">    Returns the number of parameters of a pretrained model, used to filter to only run code for sufficiently small models.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1680" href="#t1680">1680</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1681" href="#t1681">1681</a></span><span class="t">    <span class="nam">cfg</span> <span class="op">=</span> <span class="nam">get_pretrained_model_config</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1682" href="#t1682">1682</a></span><span class="t">    <span class="key">return</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">n_params</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1683" href="#t1683">1683</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1684" href="#t1684">1684</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1685" href="#t1685">1685</a></span><span class="t"><span class="com"># %% Load checkpointed model state dicts</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1686" href="#t1686">1686</a></span><span class="t"><span class="com"># The steps for which there are checkpoints in the stanford crfm models</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1687" href="#t1687">1687</a></span><span class="t"><span class="nam">STANFORD_CRFM_CHECKPOINTS</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1688" href="#t1688">1688</a></span><span class="t">    <span class="nam">list</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="num">100</span><span class="op">,</span> <span class="num">10</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1689" href="#t1689">1689</a></span><span class="t">    <span class="op">+</span> <span class="nam">list</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="num">100</span><span class="op">,</span> <span class="num">2000</span><span class="op">,</span> <span class="num">50</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1690" href="#t1690">1690</a></span><span class="t">    <span class="op">+</span> <span class="nam">list</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="num">2000</span><span class="op">,</span> <span class="num">20000</span><span class="op">,</span> <span class="num">100</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1691" href="#t1691">1691</a></span><span class="t">    <span class="op">+</span> <span class="nam">list</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="num">20000</span><span class="op">,</span> <span class="num">400000</span> <span class="op">+</span> <span class="num">1</span><span class="op">,</span> <span class="num">1000</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1692" href="#t1692">1692</a></span><span class="t"><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1693" href="#t1693">1693</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1694" href="#t1694">1694</a></span><span class="t"><span class="com"># Linearly spaced checkpoints for Pythia models, taken every 1000 steps.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1695" href="#t1695">1695</a></span><span class="t"><span class="com"># Batch size 2,097,152 tokens, so checkpoints every 2.1B tokens</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1696" href="#t1696">1696</a></span><span class="t"><span class="nam">PYTHIA_CHECKPOINTS</span> <span class="op">=</span> <span class="op">[</span><span class="num">0</span><span class="op">,</span> <span class="num">1</span><span class="op">,</span> <span class="num">2</span><span class="op">,</span> <span class="num">4</span><span class="op">,</span> <span class="num">8</span><span class="op">,</span> <span class="num">16</span><span class="op">,</span> <span class="num">32</span><span class="op">,</span> <span class="num">64</span><span class="op">,</span> <span class="num">128</span><span class="op">,</span> <span class="num">256</span><span class="op">,</span> <span class="num">512</span><span class="op">]</span> <span class="op">+</span> <span class="nam">list</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1697" href="#t1697">1697</a></span><span class="t">    <span class="nam">range</span><span class="op">(</span><span class="num">1000</span><span class="op">,</span> <span class="num">143000</span> <span class="op">+</span> <span class="num">1</span><span class="op">,</span> <span class="num">1000</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1698" href="#t1698">1698</a></span><span class="t"><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1699" href="#t1699">1699</a></span><span class="t"><span class="com"># Pythia V1 has log-spaced early checkpoints (see line above), but V0 doesn't</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1700" href="#t1700">1700</a></span><span class="t"><span class="nam">PYTHIA_V0_CHECKPOINTS</span> <span class="op">=</span> <span class="nam">list</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="num">1000</span><span class="op">,</span> <span class="num">143000</span> <span class="op">+</span> <span class="num">1</span><span class="op">,</span> <span class="num">1000</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1701" href="#t1701">1701</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1702" href="#t1702">1702</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1703" href="#t1703">1703</a></span><span class="t"><span class="key">def</span> <span class="nam">get_checkpoint_labels</span><span class="op">(</span><span class="nam">model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1704" href="#t1704">1704</a></span><span class="t">    <span class="str">"""Returns the checkpoint labels for a given model, and the label_type</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1705" href="#t1705">1705</a></span><span class="t"><span class="str">    (step or token). Raises an error for models that are not checkpointed."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1706" href="#t1706">1706</a></span><span class="t">    <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">get_official_model_name</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1707" href="#t1707">1707</a></span><span class="t">    <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"stanford-crfm/"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1708" href="#t1708">1708</a></span><span class="t">        <span class="key">return</span> <span class="nam">STANFORD_CRFM_CHECKPOINTS</span><span class="op">,</span> <span class="str">"step"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1709" href="#t1709">1709</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"EleutherAI/pythia"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1710" href="#t1710">1710</a></span><span class="t">        <span class="key">if</span> <span class="str">"v0"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1711" href="#t1711">1711</a></span><span class="t">            <span class="key">return</span> <span class="nam">PYTHIA_V0_CHECKPOINTS</span><span class="op">,</span> <span class="str">"step"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1712" href="#t1712">1712</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1713" href="#t1713">1713</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1714" href="#t1714">1714</a></span><span class="t">                <span class="str">"Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models."</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1715" href="#t1715">1715</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1716" href="#t1716">1716</a></span><span class="t">            <span class="key">return</span> <span class="nam">PYTHIA_CHECKPOINTS</span><span class="op">,</span> <span class="str">"step"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1717" href="#t1717">1717</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"NeelNanda/"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1718" href="#t1718">1718</a></span><span class="t">        <span class="nam">api</span> <span class="op">=</span> <span class="nam">HfApi</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1719" href="#t1719">1719</a></span><span class="t">        <span class="nam">files_list</span> <span class="op">=</span> <span class="nam">api</span><span class="op">.</span><span class="nam">list_repo_files</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1720" href="#t1720">1720</a></span><span class="t">            <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1721" href="#t1721">1721</a></span><span class="t">            <span class="op">**</span><span class="nam">utils</span><span class="op">.</span><span class="nam">select_compatible_kwargs</span><span class="op">(</span><span class="nam">kwargs</span><span class="op">,</span> <span class="nam">api</span><span class="op">.</span><span class="nam">list_repo_files</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1722" href="#t1722">1722</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1723" href="#t1723">1723</a></span><span class="t">        <span class="nam">labels</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1724" href="#t1724">1724</a></span><span class="t">        <span class="key">for</span> <span class="nam">file_name</span> <span class="key">in</span> <span class="nam">files_list</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1725" href="#t1725">1725</a></span><span class="t">            <span class="nam">match</span> <span class="op">=</span> <span class="nam">re</span><span class="op">.</span><span class="nam">match</span><span class="op">(</span><span class="str">r"checkpoints/.*_(\d*)\.pth"</span><span class="op">,</span> <span class="nam">file_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1726" href="#t1726">1726</a></span><span class="t">            <span class="key">if</span> <span class="nam">match</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1727" href="#t1727">1727</a></span><span class="t">                <span class="nam">labels</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">int</span><span class="op">(</span><span class="nam">match</span><span class="op">.</span><span class="nam">group</span><span class="op">(</span><span class="num">1</span><span class="op">)</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1728" href="#t1728">1728</a></span><span class="t">        <span class="key">if</span> <span class="nam">labels</span><span class="op">[</span><span class="op">-</span><span class="num">1</span><span class="op">]</span> <span class="op">></span> <span class="num">1e9</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1729" href="#t1729">1729</a></span><span class="t">            <span class="nam">label_type</span> <span class="op">=</span> <span class="str">"token"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1730" href="#t1730">1730</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1731" href="#t1731">1731</a></span><span class="t">            <span class="nam">label_type</span> <span class="op">=</span> <span class="str">"step"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1732" href="#t1732">1732</a></span><span class="t">        <span class="key">return</span> <span class="nam">labels</span><span class="op">,</span> <span class="nam">label_type</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1733" href="#t1733">1733</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1734" href="#t1734">1734</a></span><span class="t">        <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">f"Model {official_model_name} is not checkpointed."</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1735" href="#t1735">1735</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1736" href="#t1736">1736</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1737" href="#t1737">1737</a></span><span class="t"><span class="com"># %% Loading state dicts</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1738" href="#t1738">1738</a></span><span class="t"><span class="key">def</span> <span class="nam">get_pretrained_state_dict</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1739" href="#t1739">1739</a></span><span class="t">    <span class="nam">official_model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1740" href="#t1740">1740</a></span><span class="t">    <span class="nam">cfg</span><span class="op">:</span> <span class="nam">HookedTransformerConfig</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1741" href="#t1741">1741</a></span><span class="t">    <span class="nam">hf_model</span><span class="op">=</span><span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1742" href="#t1742">1742</a></span><span class="t">    <span class="nam">dtype</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">dtype</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1743" href="#t1743">1743</a></span><span class="t">    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1744" href="#t1744">1744</a></span><span class="t"><span class="op">)</span> <span class="op">-></span> <span class="nam">Dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1745" href="#t1745">1745</a></span><span class="t">    <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1746" href="#t1746">1746</a></span><span class="t"><span class="str">    Loads in the model weights for a pretrained model, and processes them to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1747" href="#t1747">1747</a></span><span class="t"><span class="str">    have the HookedTransformer parameter names and shapes. Supports checkpointed</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1748" href="#t1748">1748</a></span><span class="t"><span class="str">    models (and expects the checkpoint info to be stored in the config object)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1749" href="#t1749">1749</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1750" href="#t1750">1750</a></span><span class="t"><span class="str">    hf_model: Optionally, a HuggingFace model object. If provided, we will use</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1751" href="#t1751">1751</a></span><span class="t"><span class="str">        these weights rather than reloading the model.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1752" href="#t1752">1752</a></span><span class="t"><span class="str">    dtype: The dtype to load the HuggingFace model in.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1753" href="#t1753">1753</a></span><span class="t"><span class="str">    kwargs: Other optional arguments passed to HuggingFace's from_pretrained.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1754" href="#t1754">1754</a></span><span class="t"><span class="str">        Also given to other HuggingFace functions when compatible.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1755" href="#t1755">1755</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1756" href="#t1756">1756</a></span><span class="t">    <span class="key">if</span> <span class="str">"torch_dtype"</span> <span class="key">in</span> <span class="nam">kwargs</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1756&#x202F;&#x219B;&#x202F;1757</span><span class="annotate long">line 1756 didn't jump to line 1757, because the condition on line 1756 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1757" href="#t1757">1757</a></span><span class="t">        <span class="nam">dtype</span> <span class="op">=</span> <span class="nam">kwargs</span><span class="op">[</span><span class="str">"torch_dtype"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1758" href="#t1758">1758</a></span><span class="t">        <span class="key">del</span> <span class="nam">kwargs</span><span class="op">[</span><span class="str">"torch_dtype"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1759" href="#t1759">1759</a></span><span class="t">    <span class="key">if</span> <span class="nam">Path</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">)</span><span class="op">.</span><span class="nam">exists</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1759&#x202F;&#x219B;&#x202F;1760</span><span class="annotate long">line 1759 didn't jump to line 1760, because the condition on line 1759 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1760" href="#t1760">1760</a></span><span class="t">        <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">str</span><span class="op">(</span><span class="nam">Path</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">)</span><span class="op">.</span><span class="nam">resolve</span><span class="op">(</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1761" href="#t1761">1761</a></span><span class="t">        <span class="nam">logging</span><span class="op">.</span><span class="nam">info</span><span class="op">(</span><span class="str">f"Loading model from local path {official_model_name}"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1762" href="#t1762">1762</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1763" href="#t1763">1763</a></span><span class="t">        <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">get_official_model_name</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1764" href="#t1764">1764</a></span><span class="t">    <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="nam">NEED_REMOTE_CODE_MODELS</span><span class="op">)</span> <span class="key">and</span> <span class="key">not</span> <span class="nam">kwargs</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span>&nbsp;</span><span class="r"><span class="annotate short">1764&#x202F;&#x219B;&#x202F;1767</span><span class="annotate long">line 1764 didn't jump to line 1767, because the condition on line 1764 was never true</span></span></p>
+    <p class="pln"><span class="n"><a id="t1765" href="#t1765">1765</a></span><span class="t">        <span class="str">"trust_remote_code"</span><span class="op">,</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1766" href="#t1766">1766</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1767" href="#t1767">1767</a></span><span class="t">        <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1768" href="#t1768">1768</a></span><span class="t">            <span class="str">f"Loading model {official_model_name} state dict requires setting trust_remote_code=True"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1769" href="#t1769">1769</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1770" href="#t1770">1770</a></span><span class="t">        <span class="nam">kwargs</span><span class="op">[</span><span class="str">"trust_remote_code"</span><span class="op">]</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1771" href="#t1771">1771</a></span><span class="t">    <span class="key">if</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1772" href="#t1772">1772</a></span><span class="t">        <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"NeelNanda"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1773" href="#t1773">1773</a></span><span class="t">        <span class="key">or</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"ArthurConmy"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1774" href="#t1774">1774</a></span><span class="t">        <span class="key">or</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"Baidicoot"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1775" href="#t1775">1775</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1776" href="#t1776">1776</a></span><span class="t">        <span class="nam">api</span> <span class="op">=</span> <span class="nam">HfApi</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1777" href="#t1777">1777</a></span><span class="t">        <span class="nam">repo_files</span> <span class="op">=</span> <span class="nam">api</span><span class="op">.</span><span class="nam">list_repo_files</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1778" href="#t1778">1778</a></span><span class="t">            <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1779" href="#t1779">1779</a></span><span class="t">            <span class="op">**</span><span class="nam">utils</span><span class="op">.</span><span class="nam">select_compatible_kwargs</span><span class="op">(</span><span class="nam">kwargs</span><span class="op">,</span> <span class="nam">api</span><span class="op">.</span><span class="nam">list_repo_files</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1780" href="#t1780">1780</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1781" href="#t1781">1781</a></span><span class="t">        <span class="key">if</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">from_checkpoint</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1781&#x202F;&#x219B;&#x202F;1782</span><span class="annotate long">line 1781 didn't jump to line 1782, because the condition on line 1781 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1782" href="#t1782">1782</a></span><span class="t">            <span class="nam">file_name</span> <span class="op">=</span> <span class="nam">list</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1783" href="#t1783">1783</a></span><span class="t">                <span class="nam">filter</span><span class="op">(</span><span class="key">lambda</span> <span class="nam">x</span><span class="op">:</span> <span class="nam">x</span><span class="op">.</span><span class="nam">endswith</span><span class="op">(</span><span class="str">f"{cfg.checkpoint_value}.pth"</span><span class="op">)</span><span class="op">,</span> <span class="nam">repo_files</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1784" href="#t1784">1784</a></span><span class="t">            <span class="op">)</span><span class="op">[</span><span class="num">0</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1785" href="#t1785">1785</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1786" href="#t1786">1786</a></span><span class="t">            <span class="nam">file_name</span> <span class="op">=</span> <span class="nam">list</span><span class="op">(</span><span class="nam">filter</span><span class="op">(</span><span class="key">lambda</span> <span class="nam">x</span><span class="op">:</span> <span class="nam">x</span><span class="op">.</span><span class="nam">endswith</span><span class="op">(</span><span class="str">"final.pth"</span><span class="op">)</span><span class="op">,</span> <span class="nam">repo_files</span><span class="op">)</span><span class="op">)</span><span class="op">[</span><span class="num">0</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1787" href="#t1787">1787</a></span><span class="t">        <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">download_file_from_hf</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">,</span> <span class="nam">file_name</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1788" href="#t1788">1788</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1789" href="#t1789">1789</a></span><span class="t">        <span class="com"># Convert to dtype</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1790" href="#t1790">1790</a></span><span class="t">        <span class="nam">state_dict</span> <span class="op">=</span> <span class="op">{</span><span class="nam">k</span><span class="op">:</span> <span class="nam">v</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">dtype</span><span class="op">)</span> <span class="key">for</span> <span class="nam">k</span><span class="op">,</span> <span class="nam">v</span> <span class="key">in</span> <span class="nam">state_dict</span><span class="op">.</span><span class="nam">items</span><span class="op">(</span><span class="op">)</span><span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1791" href="#t1791">1791</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1792" href="#t1792">1792</a></span><span class="t">        <span class="key">if</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"neel-solu-old"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1793" href="#t1793">1793</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_neel_solu_old_weights</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1794" href="#t1794">1794</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"mingpt"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1795" href="#t1795">1795</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_mingpt_weights</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1796" href="#t1796">1796</a></span><span class="t">        <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1797" href="#t1797">1797</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1798" href="#t1798">1798</a></span><span class="t">        <span class="key">if</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">from_checkpoint</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1798&#x202F;&#x219B;&#x202F;1799</span><span class="annotate long">line 1798 didn't jump to line 1799, because the condition on line 1798 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1799" href="#t1799">1799</a></span><span class="t">            <span class="nam">huggingface_token</span> <span class="op">=</span> <span class="nam">os</span><span class="op">.</span><span class="nam">environ</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"HF_TOKEN"</span><span class="op">,</span> <span class="key">None</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1800" href="#t1800">1800</a></span><span class="t">            <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"stanford-crfm"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1801" href="#t1801">1801</a></span><span class="t">                <span class="nam">hf_model</span> <span class="op">=</span> <span class="nam">AutoModelForCausalLM</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1802" href="#t1802">1802</a></span><span class="t">                    <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1803" href="#t1803">1803</a></span><span class="t">                    <span class="nam">revision</span><span class="op">=</span><span class="str">f"checkpoint-{cfg.checkpoint_value}"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1804" href="#t1804">1804</a></span><span class="t">                    <span class="nam">torch_dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1805" href="#t1805">1805</a></span><span class="t">                    <span class="nam">token</span><span class="op">=</span><span class="nam">huggingface_token</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1806" href="#t1806">1806</a></span><span class="t">                    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1807" href="#t1807">1807</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1808" href="#t1808">1808</a></span><span class="t">            <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"EleutherAI/pythia"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1809" href="#t1809">1809</a></span><span class="t">                <span class="nam">hf_model</span> <span class="op">=</span> <span class="nam">AutoModelForCausalLM</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1810" href="#t1810">1810</a></span><span class="t">                    <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1811" href="#t1811">1811</a></span><span class="t">                    <span class="nam">revision</span><span class="op">=</span><span class="str">f"step{cfg.checkpoint_value}"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1812" href="#t1812">1812</a></span><span class="t">                    <span class="nam">torch_dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1813" href="#t1813">1813</a></span><span class="t">                    <span class="nam">token</span><span class="op">=</span><span class="nam">huggingface_token</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1814" href="#t1814">1814</a></span><span class="t">                    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1815" href="#t1815">1815</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1816" href="#t1816">1816</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1817" href="#t1817">1817</a></span><span class="t">            <span class="com"># Load model weights, and fold in layer norm weights</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1818" href="#t1818">1818</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1819" href="#t1819">1819</a></span><span class="t">        <span class="key">for</span> <span class="nam">param</span> <span class="key">in</span> <span class="nam">hf_model</span><span class="op">.</span><span class="nam">parameters</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1820" href="#t1820">1820</a></span><span class="t">            <span class="nam">param</span><span class="op">.</span><span class="nam">requires_grad</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1821" href="#t1821">1821</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1822" href="#t1822">1822</a></span><span class="t">        <span class="key">if</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"GPT2LMHeadModel"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1823" href="#t1823">1823</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_gpt2_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1824" href="#t1824">1824</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"GPTNeoForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1825" href="#t1825">1825</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_neo_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1826" href="#t1826">1826</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"OPTForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1827" href="#t1827">1827</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_opt_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1828" href="#t1828">1828</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"GPTJForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1828&#x202F;&#x219B;&#x202F;1829</span><span class="annotate long">line 1828 didn't jump to line 1829, because the condition on line 1828 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1829" href="#t1829">1829</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_gptj_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1830" href="#t1830">1830</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"GPTNeoXForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1831" href="#t1831">1831</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_neox_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1832" href="#t1832">1832</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"LlamaForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1832&#x202F;&#x219B;&#x202F;1833</span><span class="annotate long">line 1832 didn't jump to line 1833, because the condition on line 1832 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1833" href="#t1833">1833</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_llama_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1834" href="#t1834">1834</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"BertForMaskedLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1835" href="#t1835">1835</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_bert_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1836" href="#t1836">1836</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"T5ForConditionalGeneration"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1837" href="#t1837">1837</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_t5_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1838" href="#t1838">1838</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"MistralForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1838&#x202F;&#x219B;&#x202F;1839</span><span class="annotate long">line 1838 didn't jump to line 1839, because the condition on line 1838 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1839" href="#t1839">1839</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_mistral_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1840" href="#t1840">1840</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"MixtralForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1840&#x202F;&#x219B;&#x202F;1841</span><span class="annotate long">line 1840 didn't jump to line 1841, because the condition on line 1840 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1841" href="#t1841">1841</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_mixtral_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1842" href="#t1842">1842</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"BloomForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1842&#x202F;&#x219B;&#x202F;1844</span><span class="annotate long">line 1842 didn't jump to line 1844, because the condition on line 1842 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t1843" href="#t1843">1843</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_bloom_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1844" href="#t1844">1844</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"GPT2LMHeadCustomModel"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1845" href="#t1845">1845</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_coder_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1846" href="#t1846">1846</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"QWenLMHeadModel"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1847" href="#t1847">1847</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_qwen_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1848" href="#t1848">1848</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"Qwen2ForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1849" href="#t1849">1849</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_qwen2_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1850" href="#t1850">1850</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"PhiForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1851" href="#t1851">1851</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_phi_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1852" href="#t1852">1852</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"Phi3ForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1853" href="#t1853">1853</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_phi3_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1854" href="#t1854">1854</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"GemmaForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1855" href="#t1855">1855</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_gemma_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1856" href="#t1856">1856</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"Gemma2ForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1857" href="#t1857">1857</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_gemma_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1858" href="#t1858">1858</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1859" href="#t1859">1859</a></span><span class="t">            <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1860" href="#t1860">1860</a></span><span class="t">                <span class="str">f"Loading weights from the architecture is not currently supported: {cfg.original_architecture}, generated from model name {cfg.model_name}. Feel free to open an issue on GitHub to request this feature."</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1861" href="#t1861">1861</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1862" href="#t1862">1862</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1863" href="#t1863">1863</a></span><span class="t">        <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1864" href="#t1864">1864</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1865" href="#t1865">1865</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1866" href="#t1866">1866</a></span><span class="t"><span class="key">def</span> <span class="nam">fill_missing_keys</span><span class="op">(</span><span class="nam">model</span><span class="op">,</span> <span class="nam">state_dict</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1867" href="#t1867">1867</a></span><span class="t">    <span class="str">"""Takes in a state dict from a pretrained model, and fills in any missing keys with the default initialization.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1868" href="#t1868">1868</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1869" href="#t1869">1869</a></span><span class="t"><span class="str">    This function is assumed to be run before weights are initialized.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1870" href="#t1870">1870</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1871" href="#t1871">1871</a></span><span class="t"><span class="str">    Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1872" href="#t1872">1872</a></span><span class="t"><span class="str">        state_dict (dict): State dict from a pretrained model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1873" href="#t1873">1873</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1874" href="#t1874">1874</a></span><span class="t"><span class="str">    Returns:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1875" href="#t1875">1875</a></span><span class="t"><span class="str">        dict: State dict with missing keys filled in</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1876" href="#t1876">1876</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1877" href="#t1877">1877</a></span><span class="t">    <span class="com"># Get the default state dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1878" href="#t1878">1878</a></span><span class="t">    <span class="nam">default_state_dict</span> <span class="op">=</span> <span class="nam">model</span><span class="op">.</span><span class="nam">state_dict</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1879" href="#t1879">1879</a></span><span class="t">    <span class="com"># Get the keys that are missing from the pretrained model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1880" href="#t1880">1880</a></span><span class="t">    <span class="nam">missing_keys</span> <span class="op">=</span> <span class="nam">set</span><span class="op">(</span><span class="nam">default_state_dict</span><span class="op">.</span><span class="nam">keys</span><span class="op">(</span><span class="op">)</span><span class="op">)</span> <span class="op">-</span> <span class="nam">set</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">.</span><span class="nam">keys</span><span class="op">(</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1881" href="#t1881">1881</a></span><span class="t">    <span class="com"># Fill in the missing keys with the default initialization</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1882" href="#t1882">1882</a></span><span class="t">    <span class="key">for</span> <span class="nam">key</span> <span class="key">in</span> <span class="nam">missing_keys</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1883" href="#t1883">1883</a></span><span class="t">        <span class="key">if</span> <span class="str">"hf_model"</span> <span class="key">in</span> <span class="nam">key</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1883&#x202F;&#x219B;&#x202F;1885</span><span class="annotate long">line 1883 didn't jump to line 1885, because the condition on line 1883 was never true</span></span></p>
-    <p class="pln"><span class="n"><a id="t1884" href="#t1884">1884</a></span><span class="t">            <span class="com"># Skip keys that are from the HuggingFace model, if loading from HF.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1885" href="#t1885">1885</a></span><span class="t">            <span class="key">continue</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1886" href="#t1886">1886</a></span><span class="t">        <span class="key">if</span> <span class="str">"W_"</span> <span class="key">in</span> <span class="nam">key</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1887" href="#t1887">1887</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1888" href="#t1888">1888</a></span><span class="t">                <span class="str">"Missing key for a weight matrix in pretrained, filled in with an empty tensor: {}"</span><span class="op">.</span><span class="nam">format</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1889" href="#t1889">1889</a></span><span class="t">                    <span class="nam">key</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1890" href="#t1890">1890</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1891" href="#t1891">1891</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1892" href="#t1892">1892</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="nam">key</span><span class="op">]</span> <span class="op">=</span> <span class="nam">default_state_dict</span><span class="op">[</span><span class="nam">key</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1893" href="#t1893">1893</a></span><span class="t">    <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1894" href="#t1894">1894</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1816" href="#t1816">1816</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1817" href="#t1817">1817</a></span><span class="t">                <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">f"Checkpoints for model {official_model_name} are not supported"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1818" href="#t1818">1818</a></span><span class="t">        <span class="key">elif</span> <span class="nam">hf_model</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1818&#x202F;&#x219B;&#x202F;1846</span><span class="annotate long">line 1818 didn't jump to line 1846, because the condition on line 1818 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t1819" href="#t1819">1819</a></span><span class="t">            <span class="nam">huggingface_token</span> <span class="op">=</span> <span class="nam">os</span><span class="op">.</span><span class="nam">environ</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"HF_TOKEN"</span><span class="op">,</span> <span class="key">None</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1820" href="#t1820">1820</a></span><span class="t">            <span class="key">if</span> <span class="nam">official_model_name</span> <span class="key">in</span> <span class="nam">NON_HF_HOSTED_MODEL_NAMES</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1820&#x202F;&#x219B;&#x202F;1821</span><span class="annotate long">line 1820 didn't jump to line 1821, because the condition on line 1820 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1821" href="#t1821">1821</a></span><span class="t">                <span class="key">raise</span> <span class="nam">NotImplementedError</span><span class="op">(</span><span class="str">"Model not hosted on HuggingFace, must pass in hf_model"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1822" href="#t1822">1822</a></span><span class="t">            <span class="key">elif</span> <span class="str">"bert"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1823" href="#t1823">1823</a></span><span class="t">                <span class="nam">hf_model</span> <span class="op">=</span> <span class="nam">BertForPreTraining</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1824" href="#t1824">1824</a></span><span class="t">                    <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1825" href="#t1825">1825</a></span><span class="t">                    <span class="nam">torch_dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1826" href="#t1826">1826</a></span><span class="t">                    <span class="nam">token</span><span class="op">=</span><span class="nam">huggingface_token</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1827" href="#t1827">1827</a></span><span class="t">                    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1828" href="#t1828">1828</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1829" href="#t1829">1829</a></span><span class="t">            <span class="key">elif</span> <span class="str">"t5"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1830" href="#t1830">1830</a></span><span class="t">                <span class="nam">hf_model</span> <span class="op">=</span> <span class="nam">T5ForConditionalGeneration</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1831" href="#t1831">1831</a></span><span class="t">                    <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1832" href="#t1832">1832</a></span><span class="t">                    <span class="nam">torch_dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1833" href="#t1833">1833</a></span><span class="t">                    <span class="nam">token</span><span class="op">=</span><span class="nam">huggingface_token</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1834" href="#t1834">1834</a></span><span class="t">                    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1835" href="#t1835">1835</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1836" href="#t1836">1836</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1837" href="#t1837">1837</a></span><span class="t">                <span class="nam">hf_model</span> <span class="op">=</span> <span class="nam">AutoModelForCausalLM</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1838" href="#t1838">1838</a></span><span class="t">                    <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1839" href="#t1839">1839</a></span><span class="t">                    <span class="nam">torch_dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1840" href="#t1840">1840</a></span><span class="t">                    <span class="nam">token</span><span class="op">=</span><span class="nam">huggingface_token</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1841" href="#t1841">1841</a></span><span class="t">                    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1842" href="#t1842">1842</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1843" href="#t1843">1843</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1844" href="#t1844">1844</a></span><span class="t">            <span class="com"># Load model weights, and fold in layer norm weights</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1845" href="#t1845">1845</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1846" href="#t1846">1846</a></span><span class="t">        <span class="key">for</span> <span class="nam">param</span> <span class="key">in</span> <span class="nam">hf_model</span><span class="op">.</span><span class="nam">parameters</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1847" href="#t1847">1847</a></span><span class="t">            <span class="nam">param</span><span class="op">.</span><span class="nam">requires_grad</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1848" href="#t1848">1848</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1849" href="#t1849">1849</a></span><span class="t">        <span class="key">if</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"GPT2LMHeadModel"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1850" href="#t1850">1850</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_gpt2_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1851" href="#t1851">1851</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"GPTNeoForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1852" href="#t1852">1852</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_neo_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1853" href="#t1853">1853</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"OPTForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1854" href="#t1854">1854</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_opt_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1855" href="#t1855">1855</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"GPTJForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1855&#x202F;&#x219B;&#x202F;1856</span><span class="annotate long">line 1855 didn't jump to line 1856, because the condition on line 1855 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1856" href="#t1856">1856</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_gptj_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1857" href="#t1857">1857</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"GPTNeoXForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1858" href="#t1858">1858</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_neox_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1859" href="#t1859">1859</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"LlamaForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1859&#x202F;&#x219B;&#x202F;1860</span><span class="annotate long">line 1859 didn't jump to line 1860, because the condition on line 1859 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1860" href="#t1860">1860</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_llama_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1861" href="#t1861">1861</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"BertForMaskedLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1862" href="#t1862">1862</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_bert_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1863" href="#t1863">1863</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"T5ForConditionalGeneration"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1864" href="#t1864">1864</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_t5_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1865" href="#t1865">1865</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"MistralForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1865&#x202F;&#x219B;&#x202F;1866</span><span class="annotate long">line 1865 didn't jump to line 1866, because the condition on line 1865 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1866" href="#t1866">1866</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_mistral_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1867" href="#t1867">1867</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"MixtralForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1867&#x202F;&#x219B;&#x202F;1868</span><span class="annotate long">line 1867 didn't jump to line 1868, because the condition on line 1867 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1868" href="#t1868">1868</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_mixtral_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1869" href="#t1869">1869</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"BloomForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1870" href="#t1870">1870</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_bloom_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1871" href="#t1871">1871</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"GPT2LMHeadCustomModel"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1871&#x202F;&#x219B;&#x202F;1872</span><span class="annotate long">line 1871 didn't jump to line 1872, because the condition on line 1871 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1872" href="#t1872">1872</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_coder_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1873" href="#t1873">1873</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"QWenLMHeadModel"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1873&#x202F;&#x219B;&#x202F;1874</span><span class="annotate long">line 1873 didn't jump to line 1874, because the condition on line 1873 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1874" href="#t1874">1874</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_qwen_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1875" href="#t1875">1875</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"Qwen2ForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1875&#x202F;&#x219B;&#x202F;1877</span><span class="annotate long">line 1875 didn't jump to line 1877, because the condition on line 1875 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t1876" href="#t1876">1876</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_qwen2_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1877" href="#t1877">1877</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"PhiForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1878" href="#t1878">1878</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_phi_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1879" href="#t1879">1879</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"Phi3ForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1880" href="#t1880">1880</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_phi3_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1881" href="#t1881">1881</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"GemmaForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1882" href="#t1882">1882</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_gemma_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1883" href="#t1883">1883</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"Gemma2ForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1884" href="#t1884">1884</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_gemma_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1885" href="#t1885">1885</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1886" href="#t1886">1886</a></span><span class="t">            <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1887" href="#t1887">1887</a></span><span class="t">                <span class="str">f"Loading weights from the architecture is not currently supported: {cfg.original_architecture}, generated from model name {cfg.model_name}. Feel free to open an issue on GitHub to request this feature."</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1888" href="#t1888">1888</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1889" href="#t1889">1889</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1890" href="#t1890">1890</a></span><span class="t">        <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1891" href="#t1891">1891</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1892" href="#t1892">1892</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1893" href="#t1893">1893</a></span><span class="t"><span class="key">def</span> <span class="nam">fill_missing_keys</span><span class="op">(</span><span class="nam">model</span><span class="op">,</span> <span class="nam">state_dict</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1894" href="#t1894">1894</a></span><span class="t">    <span class="str">"""Takes in a state dict from a pretrained model, and fills in any missing keys with the default initialization.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1895" href="#t1895">1895</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1896" href="#t1896">1896</a></span><span class="t"><span class="op">@</span><span class="nam">dataclasses</span><span class="op">.</span><span class="nam">dataclass</span>&nbsp;</span><span class="r"><span class="annotate short">1896&#x202F;&#x219B;&#x202F;1898</span><span class="annotate long">line 1896 didn't jump to line 1898, because </span></span></p>
-    <p class="run"><span class="n"><a id="t1897" href="#t1897">1897</a></span><span class="t"><span class="key">class</span> <span class="nam">Config</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1898" href="#t1898">1898</a></span><span class="t">    <span class="nam">d_model</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">768</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1899" href="#t1899">1899</a></span><span class="t">    <span class="nam">debug</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1900" href="#t1900">1900</a></span><span class="t">    <span class="nam">layer_norm_eps</span><span class="op">:</span> <span class="nam">float</span> <span class="op">=</span> <span class="num">1e-5</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1901" href="#t1901">1901</a></span><span class="t">    <span class="nam">d_vocab</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">50257</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1902" href="#t1902">1902</a></span><span class="t">    <span class="nam">init_range</span><span class="op">:</span> <span class="nam">float</span> <span class="op">=</span> <span class="num">0.02</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1903" href="#t1903">1903</a></span><span class="t">    <span class="nam">n_ctx</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">1024</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1904" href="#t1904">1904</a></span><span class="t">    <span class="nam">d_head</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">64</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1905" href="#t1905">1905</a></span><span class="t">    <span class="nam">d_mlp</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">3072</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1906" href="#t1906">1906</a></span><span class="t">    <span class="nam">n_heads</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">12</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1907" href="#t1907">1907</a></span><span class="t">    <span class="nam">n_layers</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">12</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1908" href="#t1908">1908</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1909" href="#t1909">1909</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1910" href="#t1910">1910</a></span><span class="t"><span class="com"># Returns the configuration parameters of the model as a basic Config dataclass</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1911" href="#t1911">1911</a></span><span class="t"><span class="key">def</span> <span class="nam">get_basic_config</span><span class="op">(</span><span class="nam">model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Config</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1912" href="#t1912">1912</a></span><span class="t">    <span class="key">return</span> <span class="nam">Config</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1913" href="#t1913">1913</a></span><span class="t">        <span class="op">**</span><span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1914" href="#t1914">1914</a></span><span class="t">            <span class="nam">k</span><span class="op">:</span> <span class="nam">v</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1915" href="#t1915">1915</a></span><span class="t">            <span class="key">for</span> <span class="nam">k</span><span class="op">,</span> <span class="nam">v</span> <span class="key">in</span> <span class="nam">get_pretrained_model_config</span><span class="op">(</span><span class="nam">model_name</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span><span class="op">.</span><span class="nam">to_dict</span><span class="op">(</span><span class="op">)</span><span class="op">.</span><span class="nam">items</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1916" href="#t1916">1916</a></span><span class="t">            <span class="key">if</span> <span class="nam">k</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1917" href="#t1917">1917</a></span><span class="t">            <span class="key">in</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1918" href="#t1918">1918</a></span><span class="t">                <span class="str">"d_model"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1919" href="#t1919">1919</a></span><span class="t">                <span class="str">"debug"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1920" href="#t1920">1920</a></span><span class="t">                <span class="str">"layer_norm_eps"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1921" href="#t1921">1921</a></span><span class="t">                <span class="str">"d_vocab"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1922" href="#t1922">1922</a></span><span class="t">                <span class="str">"init_range"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1923" href="#t1923">1923</a></span><span class="t">                <span class="str">"n_ctx"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1924" href="#t1924">1924</a></span><span class="t">                <span class="str">"d_head"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1925" href="#t1925">1925</a></span><span class="t">                <span class="str">"d_mlp"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1926" href="#t1926">1926</a></span><span class="t">                <span class="str">"n_heads"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1927" href="#t1927">1927</a></span><span class="t">                <span class="str">"n_layers"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1928" href="#t1928">1928</a></span><span class="t">            <span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1929" href="#t1929">1929</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1930" href="#t1930">1930</a></span><span class="t">    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1896" href="#t1896">1896</a></span><span class="t"><span class="str">    This function is assumed to be run before weights are initialized.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1897" href="#t1897">1897</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1898" href="#t1898">1898</a></span><span class="t"><span class="str">    Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1899" href="#t1899">1899</a></span><span class="t"><span class="str">        state_dict (dict): State dict from a pretrained model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1900" href="#t1900">1900</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1901" href="#t1901">1901</a></span><span class="t"><span class="str">    Returns:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1902" href="#t1902">1902</a></span><span class="t"><span class="str">        dict: State dict with missing keys filled in</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1903" href="#t1903">1903</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1904" href="#t1904">1904</a></span><span class="t">    <span class="com"># Get the default state dict</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1905" href="#t1905">1905</a></span><span class="t">    <span class="nam">default_state_dict</span> <span class="op">=</span> <span class="nam">model</span><span class="op">.</span><span class="nam">state_dict</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1906" href="#t1906">1906</a></span><span class="t">    <span class="com"># Get the keys that are missing from the pretrained model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1907" href="#t1907">1907</a></span><span class="t">    <span class="nam">missing_keys</span> <span class="op">=</span> <span class="nam">set</span><span class="op">(</span><span class="nam">default_state_dict</span><span class="op">.</span><span class="nam">keys</span><span class="op">(</span><span class="op">)</span><span class="op">)</span> <span class="op">-</span> <span class="nam">set</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">.</span><span class="nam">keys</span><span class="op">(</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1908" href="#t1908">1908</a></span><span class="t">    <span class="com"># Fill in the missing keys with the default initialization</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1909" href="#t1909">1909</a></span><span class="t">    <span class="key">for</span> <span class="nam">key</span> <span class="key">in</span> <span class="nam">missing_keys</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1910" href="#t1910">1910</a></span><span class="t">        <span class="key">if</span> <span class="str">"hf_model"</span> <span class="key">in</span> <span class="nam">key</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1910&#x202F;&#x219B;&#x202F;1912</span><span class="annotate long">line 1910 didn't jump to line 1912, because the condition on line 1910 was never true</span></span></p>
+    <p class="pln"><span class="n"><a id="t1911" href="#t1911">1911</a></span><span class="t">            <span class="com"># Skip keys that are from the HuggingFace model, if loading from HF.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1912" href="#t1912">1912</a></span><span class="t">            <span class="key">continue</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1913" href="#t1913">1913</a></span><span class="t">        <span class="key">if</span> <span class="str">"W_"</span> <span class="key">in</span> <span class="nam">key</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1914" href="#t1914">1914</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1915" href="#t1915">1915</a></span><span class="t">                <span class="str">"Missing key for a weight matrix in pretrained, filled in with an empty tensor: {}"</span><span class="op">.</span><span class="nam">format</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1916" href="#t1916">1916</a></span><span class="t">                    <span class="nam">key</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1917" href="#t1917">1917</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1918" href="#t1918">1918</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1919" href="#t1919">1919</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="nam">key</span><span class="op">]</span> <span class="op">=</span> <span class="nam">default_state_dict</span><span class="op">[</span><span class="nam">key</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1920" href="#t1920">1920</a></span><span class="t">    <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1921" href="#t1921">1921</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1922" href="#t1922">1922</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1923" href="#t1923">1923</a></span><span class="t"><span class="op">@</span><span class="nam">dataclasses</span><span class="op">.</span><span class="nam">dataclass</span>&nbsp;</span><span class="r"><span class="annotate short">1923&#x202F;&#x219B;&#x202F;1925</span><span class="annotate long">line 1923 didn't jump to line 1925, because </span></span></p>
+    <p class="run"><span class="n"><a id="t1924" href="#t1924">1924</a></span><span class="t"><span class="key">class</span> <span class="nam">Config</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1925" href="#t1925">1925</a></span><span class="t">    <span class="nam">d_model</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">768</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1926" href="#t1926">1926</a></span><span class="t">    <span class="nam">debug</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1927" href="#t1927">1927</a></span><span class="t">    <span class="nam">layer_norm_eps</span><span class="op">:</span> <span class="nam">float</span> <span class="op">=</span> <span class="num">1e-5</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1928" href="#t1928">1928</a></span><span class="t">    <span class="nam">d_vocab</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">50257</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1929" href="#t1929">1929</a></span><span class="t">    <span class="nam">init_range</span><span class="op">:</span> <span class="nam">float</span> <span class="op">=</span> <span class="num">0.02</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1930" href="#t1930">1930</a></span><span class="t">    <span class="nam">n_ctx</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">1024</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1931" href="#t1931">1931</a></span><span class="t">    <span class="nam">d_head</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">64</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1932" href="#t1932">1932</a></span><span class="t">    <span class="nam">d_mlp</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">3072</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1933" href="#t1933">1933</a></span><span class="t">    <span class="nam">n_heads</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">12</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1934" href="#t1934">1934</a></span><span class="t">    <span class="nam">n_layers</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">12</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1935" href="#t1935">1935</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1936" href="#t1936">1936</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1937" href="#t1937">1937</a></span><span class="t"><span class="com"># Returns the configuration parameters of the model as a basic Config dataclass</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1938" href="#t1938">1938</a></span><span class="t"><span class="key">def</span> <span class="nam">get_basic_config</span><span class="op">(</span><span class="nam">model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Config</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1939" href="#t1939">1939</a></span><span class="t">    <span class="key">return</span> <span class="nam">Config</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1940" href="#t1940">1940</a></span><span class="t">        <span class="op">**</span><span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1941" href="#t1941">1941</a></span><span class="t">            <span class="nam">k</span><span class="op">:</span> <span class="nam">v</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1942" href="#t1942">1942</a></span><span class="t">            <span class="key">for</span> <span class="nam">k</span><span class="op">,</span> <span class="nam">v</span> <span class="key">in</span> <span class="nam">get_pretrained_model_config</span><span class="op">(</span><span class="nam">model_name</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span><span class="op">.</span><span class="nam">to_dict</span><span class="op">(</span><span class="op">)</span><span class="op">.</span><span class="nam">items</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1943" href="#t1943">1943</a></span><span class="t">            <span class="key">if</span> <span class="nam">k</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1944" href="#t1944">1944</a></span><span class="t">            <span class="key">in</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1945" href="#t1945">1945</a></span><span class="t">                <span class="str">"d_model"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1946" href="#t1946">1946</a></span><span class="t">                <span class="str">"debug"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1947" href="#t1947">1947</a></span><span class="t">                <span class="str">"layer_norm_eps"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1948" href="#t1948">1948</a></span><span class="t">                <span class="str">"d_vocab"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1949" href="#t1949">1949</a></span><span class="t">                <span class="str">"init_range"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1950" href="#t1950">1950</a></span><span class="t">                <span class="str">"n_ctx"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1951" href="#t1951">1951</a></span><span class="t">                <span class="str">"d_head"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1952" href="#t1952">1952</a></span><span class="t">                <span class="str">"d_mlp"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1953" href="#t1953">1953</a></span><span class="t">                <span class="str">"n_heads"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1954" href="#t1954">1954</a></span><span class="t">                <span class="str">"n_layers"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1955" href="#t1955">1955</a></span><span class="t">            <span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1956" href="#t1956">1956</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1957" href="#t1957">1957</a></span><span class="t">    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
 </main>
 <footer>
     <div class="content">
@@ -2021,7 +2048,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_past_key_value_caching_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_past_key_value_caching_py.html b/_static/coverage/d_af97b5493da09a14_past_key_value_caching_py.html
index 3cc8b85b5..281fef2cd 100644
--- a/_static/coverage/d_af97b5493da09a14_past_key_value_caching_py.html
+++ b/_static/coverage/d_af97b5493da09a14_past_key_value_caching_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_patching_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -206,7 +206,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_patching_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_patching_py.html b/_static/coverage/d_af97b5493da09a14_patching_py.html
index a2652d699..915896066 100644
--- a/_static/coverage/d_af97b5493da09a14_patching_py.html
+++ b/_static/coverage/d_af97b5493da09a14_patching_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_712808f24eb400fe___init___py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -833,7 +833,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_712808f24eb400fe___init___py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_train_py.html b/_static/coverage/d_af97b5493da09a14_train_py.html
index 2604e2873..bbc0387a2 100644
--- a/_static/coverage/d_af97b5493da09a14_train_py.html
+++ b/_static/coverage/d_af97b5493da09a14_train_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_b2114f845e0399b7___init___py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -245,7 +245,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_b2114f845e0399b7___init___py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_utils_py.html b/_static/coverage/d_af97b5493da09a14_utils_py.html
index 0c212a1c4..26406e6d8 100644
--- a/_static/coverage/d_af97b5493da09a14_utils_py.html
+++ b/_static/coverage/d_af97b5493da09a14_utils_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="index.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -1381,7 +1381,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="index.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_b2114f845e0399b7___init___py.html b/_static/coverage/d_b2114f845e0399b7___init___py.html
index 00b8e82bb..2c6a2ccde 100644
--- a/_static/coverage/d_b2114f845e0399b7___init___py.html
+++ b/_static/coverage/d_b2114f845e0399b7___init___py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_b2114f845e0399b7_activation_functions_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -91,7 +91,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_b2114f845e0399b7_activation_functions_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_b2114f845e0399b7_activation_functions_py.html b/_static/coverage/d_b2114f845e0399b7_activation_functions_py.html
index 05cc5cc0f..642c4fe3c 100644
--- a/_static/coverage/d_b2114f845e0399b7_activation_functions_py.html
+++ b/_static/coverage/d_b2114f845e0399b7_activation_functions_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_b2114f845e0399b7_addmm_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -117,7 +117,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_b2114f845e0399b7_addmm_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_b2114f845e0399b7_addmm_py.html b/_static/coverage/d_b2114f845e0399b7_addmm_py.html
index 8802ddd1a..ca4215027 100644
--- a/_static/coverage/d_b2114f845e0399b7_addmm_py.html
+++ b/_static/coverage/d_b2114f845e0399b7_addmm_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_b2114f845e0399b7_attention_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -126,7 +126,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_b2114f845e0399b7_attention_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_b2114f845e0399b7_attention_py.html b/_static/coverage/d_b2114f845e0399b7_attention_py.html
index 9594fdaad..6b6fa9bde 100644
--- a/_static/coverage/d_b2114f845e0399b7_attention_py.html
+++ b/_static/coverage/d_b2114f845e0399b7_attention_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_b2114f845e0399b7_devices_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -133,7 +133,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_b2114f845e0399b7_devices_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_b2114f845e0399b7_devices_py.html b/_static/coverage/d_b2114f845e0399b7_devices_py.html
index 9192f259d..8d658ccfe 100644
--- a/_static/coverage/d_b2114f845e0399b7_devices_py.html
+++ b/_static/coverage/d_b2114f845e0399b7_devices_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_utils_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -166,7 +166,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_utils_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7___init___py.html b/_static/coverage/d_c1ea89878f9b2ac7___init___py.html
index 155f673bd..242d106f0 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7___init___py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7___init___py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_bert_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -111,7 +111,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_bert_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_bert_py.html b/_static/coverage/d_c1ea89878f9b2ac7_bert_py.html
index bd4b43bad..495578cc8 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_bert_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_bert_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_bloom_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -156,7 +156,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_bloom_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_bloom_py.html b/_static/coverage/d_c1ea89878f9b2ac7_bloom_py.html
index 8abbc960b..a45524ac8 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_bloom_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_bloom_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_coder_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -148,7 +148,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_coder_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_coder_py.html b/_static/coverage/d_c1ea89878f9b2ac7_coder_py.html
index dbafdcf86..8d253ec64 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_coder_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_coder_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_gemma_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -154,7 +154,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_gemma_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_gemma_py.html b/_static/coverage/d_c1ea89878f9b2ac7_gemma_py.html
index 9b40eeb20..6081147e7 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_gemma_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_gemma_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_gpt2_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -186,7 +186,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_gpt2_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_gpt2_py.html b/_static/coverage/d_c1ea89878f9b2ac7_gpt2_py.html
index 20d7bb9f9..9dbe27cc1 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_gpt2_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_gpt2_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_gptj_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -151,7 +151,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_gptj_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_gptj_py.html b/_static/coverage/d_c1ea89878f9b2ac7_gptj_py.html
index d55981ee9..7a4d12edb 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_gptj_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_gptj_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_llama_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -141,7 +141,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_llama_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_llama_py.html b/_static/coverage/d_c1ea89878f9b2ac7_llama_py.html
index f87463c3e..719a4449c 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_llama_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_llama_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_mingpt_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -187,7 +187,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_mingpt_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_mingpt_py.html b/_static/coverage/d_c1ea89878f9b2ac7_mingpt_py.html
index a80e082d2..499d55893 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_mingpt_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_mingpt_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_mistral_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -154,7 +154,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_mistral_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_mistral_py.html b/_static/coverage/d_c1ea89878f9b2ac7_mistral_py.html
index ba347b3c5..49b671ad4 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_mistral_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_mistral_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_mixtral_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -148,7 +148,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_mixtral_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_mixtral_py.html b/_static/coverage/d_c1ea89878f9b2ac7_mixtral_py.html
index 10e837abf..d09db4200 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_mixtral_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_mixtral_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_nanogpt_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -164,7 +164,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_nanogpt_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_nanogpt_py.html b/_static/coverage/d_c1ea89878f9b2ac7_nanogpt_py.html
index af9e32c6b..1141cd1d4 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_nanogpt_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_nanogpt_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_neel_solu_old_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -179,7 +179,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_neel_solu_old_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_neel_solu_old_py.html b/_static/coverage/d_c1ea89878f9b2ac7_neel_solu_old_py.html
index d25ae733e..75968fce4 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_neel_solu_old_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_neel_solu_old_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_neo_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -129,7 +129,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_neo_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_neo_py.html b/_static/coverage/d_c1ea89878f9b2ac7_neo_py.html
index a8325657b..48329a390 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_neo_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_neo_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_neox_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -140,7 +140,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_neox_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_neox_py.html b/_static/coverage/d_c1ea89878f9b2ac7_neox_py.html
index ece3ee9bd..5af4f1b97 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_neox_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_neox_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_opt_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -150,7 +150,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_opt_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_opt_py.html b/_static/coverage/d_c1ea89878f9b2ac7_opt_py.html
index ed46de3ba..9802dbf71 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_opt_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_opt_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_phi_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -175,7 +175,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_phi_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_phi3_py.html b/_static/coverage/d_c1ea89878f9b2ac7_phi3_py.html
index 93f0f430d..120c9e77f 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_phi3_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_phi3_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_qwen_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -147,7 +147,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_qwen_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_phi_py.html b/_static/coverage/d_c1ea89878f9b2ac7_phi_py.html
index f2267d8a2..738f6d11d 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_phi_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_phi_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_phi3_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -155,7 +155,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_phi3_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_qwen2_py.html b/_static/coverage/d_c1ea89878f9b2ac7_qwen2_py.html
index 48dbcd55d..a1426450a 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_qwen2_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_qwen2_py.html
@@ -2,7 +2,7 @@
 <html>
 <head>
     <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
-    <title>Coverage for transformer_lens/pretrained/weight_conversions/qwen2.py: 12%</title>
+    <title>Coverage for transformer_lens/pretrained/weight_conversions/qwen2.py: 100%</title>
     <link rel="icon" sizes="32x32" href="favicon_32.png">
     <link rel="stylesheet" href="style.css" type="text/css">
     <script type="text/javascript" src="coverage_html.js" defer></script>
@@ -12,7 +12,7 @@
     <div class="content">
         <h1>
             <span class="text">Coverage for </span><b>transformer_lens/pretrained/weight_conversions/qwen2.py</b>:
-            <span class="pc_cov">12%</span>
+            <span class="pc_cov">100%</span>
         </h1>
         <aside id="help_panel_wrapper">
             <input id="help_panel_state" type="checkbox">
@@ -56,8 +56,8 @@ <h1>
         </aside>
         <h2>
             <span class="text">41 statements &nbsp;</span>
-            <button type="button" class="run button_toggle_run" value="run" data-shortcut="r" title="Toggle lines run">5<span class="text"> run</span></button>
-            <button type="button" class="mis show_mis button_toggle_mis" value="mis" data-shortcut="m" title="Toggle lines missing">36<span class="text"> missing</span></button>
+            <button type="button" class="run button_toggle_run" value="run" data-shortcut="r" title="Toggle lines run">41<span class="text"> run</span></button>
+            <button type="button" class="mis show_mis button_toggle_mis" value="mis" data-shortcut="m" title="Toggle lines missing">0<span class="text"> missing</span></button>
             <button type="button" class="exc show_exc button_toggle_exc" value="exc" data-shortcut="x" title="Toggle lines excluded">0<span class="text"> excluded</span></button>
             <button type="button" class="par run show_par button_toggle_par" value="par" data-shortcut="p" title="Toggle lines partially run">0<span class="text"> partial</span></button>
         </h2>
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_t5_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -92,70 +92,70 @@ <h2>
     <p class="pln"><span class="n"><a id="t8" href="#t8">8</a></span><span class="t">    <span class="com"># Note that this method is also applied for Qwen1.5 models, since they</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t9" href="#t9">9</a></span><span class="t">    <span class="com"># have architecture type Qwen2ForCausalLM.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t10" href="#t10">10</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t11" href="#t11">11</a></span><span class="t">    <span class="nam">state_dict</span> <span class="op">=</span> <span class="op">{</span><span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t11" href="#t11">11</a></span><span class="t">    <span class="nam">state_dict</span> <span class="op">=</span> <span class="op">{</span><span class="op">}</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t12" href="#t12">12</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t13" href="#t13">13</a></span><span class="t">    <span class="nam">state_dict</span><span class="op">[</span><span class="str">"embed.W_E"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">qwen</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">embed_tokens</span><span class="op">.</span><span class="nam">weight</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t13" href="#t13">13</a></span><span class="t">    <span class="nam">state_dict</span><span class="op">[</span><span class="str">"embed.W_E"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">qwen</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">embed_tokens</span><span class="op">.</span><span class="nam">weight</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t14" href="#t14">14</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t15" href="#t15">15</a></span><span class="t">    <span class="key">assert</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">d_mlp</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>  <span class="com"># keep mypy happy</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t15" href="#t15">15</a></span><span class="t">    <span class="key">assert</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">d_mlp</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>  <span class="com"># keep mypy happy</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t16" href="#t16">16</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t17" href="#t17">17</a></span><span class="t">    <span class="key">for</span> <span class="nam">l</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t18" href="#t18">18</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln1.w"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">qwen</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">layers</span><span class="op">[</span><span class="nam">l</span><span class="op">]</span><span class="op">.</span><span class="nam">input_layernorm</span><span class="op">.</span><span class="nam">weight</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t17" href="#t17">17</a></span><span class="t">    <span class="key">for</span> <span class="nam">l</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t18" href="#t18">18</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln1.w"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">qwen</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">layers</span><span class="op">[</span><span class="nam">l</span><span class="op">]</span><span class="op">.</span><span class="nam">input_layernorm</span><span class="op">.</span><span class="nam">weight</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t19" href="#t19">19</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t20" href="#t20">20</a></span><span class="t">        <span class="nam">W_Q</span> <span class="op">=</span> <span class="nam">qwen</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">layers</span><span class="op">[</span><span class="nam">l</span><span class="op">]</span><span class="op">.</span><span class="nam">self_attn</span><span class="op">.</span><span class="nam">q_proj</span><span class="op">.</span><span class="nam">weight</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t21" href="#t21">21</a></span><span class="t">        <span class="nam">W_K</span> <span class="op">=</span> <span class="nam">qwen</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">layers</span><span class="op">[</span><span class="nam">l</span><span class="op">]</span><span class="op">.</span><span class="nam">self_attn</span><span class="op">.</span><span class="nam">k_proj</span><span class="op">.</span><span class="nam">weight</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t22" href="#t22">22</a></span><span class="t">        <span class="nam">W_V</span> <span class="op">=</span> <span class="nam">qwen</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">layers</span><span class="op">[</span><span class="nam">l</span><span class="op">]</span><span class="op">.</span><span class="nam">self_attn</span><span class="op">.</span><span class="nam">v_proj</span><span class="op">.</span><span class="nam">weight</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t23" href="#t23">23</a></span><span class="t">        <span class="nam">W_Q</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span><span class="nam">W_Q</span><span class="op">,</span> <span class="str">"(n h) m->n m h"</span><span class="op">,</span> <span class="nam">n</span><span class="op">=</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_heads</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t24" href="#t24">24</a></span><span class="t">        <span class="nam">W_K</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span><span class="nam">W_K</span><span class="op">,</span> <span class="str">"(n h) m->n m h"</span><span class="op">,</span> <span class="nam">n</span><span class="op">=</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_key_value_heads</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t25" href="#t25">25</a></span><span class="t">        <span class="nam">W_V</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span><span class="nam">W_V</span><span class="op">,</span> <span class="str">"(n h) m->n m h"</span><span class="op">,</span> <span class="nam">n</span><span class="op">=</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_key_value_heads</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t20" href="#t20">20</a></span><span class="t">        <span class="nam">W_Q</span> <span class="op">=</span> <span class="nam">qwen</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">layers</span><span class="op">[</span><span class="nam">l</span><span class="op">]</span><span class="op">.</span><span class="nam">self_attn</span><span class="op">.</span><span class="nam">q_proj</span><span class="op">.</span><span class="nam">weight</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t21" href="#t21">21</a></span><span class="t">        <span class="nam">W_K</span> <span class="op">=</span> <span class="nam">qwen</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">layers</span><span class="op">[</span><span class="nam">l</span><span class="op">]</span><span class="op">.</span><span class="nam">self_attn</span><span class="op">.</span><span class="nam">k_proj</span><span class="op">.</span><span class="nam">weight</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t22" href="#t22">22</a></span><span class="t">        <span class="nam">W_V</span> <span class="op">=</span> <span class="nam">qwen</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">layers</span><span class="op">[</span><span class="nam">l</span><span class="op">]</span><span class="op">.</span><span class="nam">self_attn</span><span class="op">.</span><span class="nam">v_proj</span><span class="op">.</span><span class="nam">weight</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t23" href="#t23">23</a></span><span class="t">        <span class="nam">W_Q</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span><span class="nam">W_Q</span><span class="op">,</span> <span class="str">"(n h) m->n m h"</span><span class="op">,</span> <span class="nam">n</span><span class="op">=</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_heads</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t24" href="#t24">24</a></span><span class="t">        <span class="nam">W_K</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span><span class="nam">W_K</span><span class="op">,</span> <span class="str">"(n h) m->n m h"</span><span class="op">,</span> <span class="nam">n</span><span class="op">=</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_key_value_heads</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t25" href="#t25">25</a></span><span class="t">        <span class="nam">W_V</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span><span class="nam">W_V</span><span class="op">,</span> <span class="str">"(n h) m->n m h"</span><span class="op">,</span> <span class="nam">n</span><span class="op">=</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_key_value_heads</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t26" href="#t26">26</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t27" href="#t27">27</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_Q"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">W_Q</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t28" href="#t28">28</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn._W_K"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">W_K</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t29" href="#t29">29</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn._W_V"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">W_V</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t27" href="#t27">27</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_Q"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">W_Q</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t28" href="#t28">28</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn._W_K"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">W_K</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t29" href="#t29">29</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn._W_V"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">W_V</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t30" href="#t30">30</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t31" href="#t31">31</a></span><span class="t">        <span class="nam">b_Q</span> <span class="op">=</span> <span class="nam">qwen</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">layers</span><span class="op">[</span><span class="nam">l</span><span class="op">]</span><span class="op">.</span><span class="nam">self_attn</span><span class="op">.</span><span class="nam">q_proj</span><span class="op">.</span><span class="nam">bias</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t32" href="#t32">32</a></span><span class="t">        <span class="nam">b_Q</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t31" href="#t31">31</a></span><span class="t">        <span class="nam">b_Q</span> <span class="op">=</span> <span class="nam">qwen</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">layers</span><span class="op">[</span><span class="nam">l</span><span class="op">]</span><span class="op">.</span><span class="nam">self_attn</span><span class="op">.</span><span class="nam">q_proj</span><span class="op">.</span><span class="nam">bias</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t32" href="#t32">32</a></span><span class="t">        <span class="nam">b_Q</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t33" href="#t33">33</a></span><span class="t">            <span class="nam">b_Q</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t34" href="#t34">34</a></span><span class="t">            <span class="str">"(n_head d_head) -> n_head d_head"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t35" href="#t35">35</a></span><span class="t">            <span class="nam">n_head</span><span class="op">=</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t36" href="#t36">36</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t37" href="#t37">37</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t38" href="#t38">38</a></span><span class="t">        <span class="nam">b_K</span> <span class="op">=</span> <span class="nam">qwen</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">layers</span><span class="op">[</span><span class="nam">l</span><span class="op">]</span><span class="op">.</span><span class="nam">self_attn</span><span class="op">.</span><span class="nam">k_proj</span><span class="op">.</span><span class="nam">bias</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t39" href="#t39">39</a></span><span class="t">        <span class="nam">b_K</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t38" href="#t38">38</a></span><span class="t">        <span class="nam">b_K</span> <span class="op">=</span> <span class="nam">qwen</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">layers</span><span class="op">[</span><span class="nam">l</span><span class="op">]</span><span class="op">.</span><span class="nam">self_attn</span><span class="op">.</span><span class="nam">k_proj</span><span class="op">.</span><span class="nam">bias</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t39" href="#t39">39</a></span><span class="t">        <span class="nam">b_K</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t40" href="#t40">40</a></span><span class="t">            <span class="nam">b_K</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t41" href="#t41">41</a></span><span class="t">            <span class="str">"(n_head d_head) -> n_head d_head"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t42" href="#t42">42</a></span><span class="t">            <span class="nam">n_head</span><span class="op">=</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_key_value_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t43" href="#t43">43</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t44" href="#t44">44</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t45" href="#t45">45</a></span><span class="t">        <span class="nam">b_V</span> <span class="op">=</span> <span class="nam">qwen</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">layers</span><span class="op">[</span><span class="nam">l</span><span class="op">]</span><span class="op">.</span><span class="nam">self_attn</span><span class="op">.</span><span class="nam">v_proj</span><span class="op">.</span><span class="nam">bias</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t46" href="#t46">46</a></span><span class="t">        <span class="nam">b_V</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t45" href="#t45">45</a></span><span class="t">        <span class="nam">b_V</span> <span class="op">=</span> <span class="nam">qwen</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">layers</span><span class="op">[</span><span class="nam">l</span><span class="op">]</span><span class="op">.</span><span class="nam">self_attn</span><span class="op">.</span><span class="nam">v_proj</span><span class="op">.</span><span class="nam">bias</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t46" href="#t46">46</a></span><span class="t">        <span class="nam">b_V</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t47" href="#t47">47</a></span><span class="t">            <span class="nam">b_V</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t48" href="#t48">48</a></span><span class="t">            <span class="str">"(n_head d_head) -> n_head d_head"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t49" href="#t49">49</a></span><span class="t">            <span class="nam">n_head</span><span class="op">=</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_key_value_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t50" href="#t50">50</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t51" href="#t51">51</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t52" href="#t52">52</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_Q"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">b_Q</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t53" href="#t53">53</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn._b_K"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">b_K</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t54" href="#t54">54</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn._b_V"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">b_V</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t52" href="#t52">52</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_Q"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">b_Q</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t53" href="#t53">53</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn._b_K"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">b_K</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t54" href="#t54">54</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn._b_V"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">b_V</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t55" href="#t55">55</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t56" href="#t56">56</a></span><span class="t">        <span class="nam">W_O</span> <span class="op">=</span> <span class="nam">qwen</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">layers</span><span class="op">[</span><span class="nam">l</span><span class="op">]</span><span class="op">.</span><span class="nam">self_attn</span><span class="op">.</span><span class="nam">o_proj</span><span class="op">.</span><span class="nam">weight</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t57" href="#t57">57</a></span><span class="t">        <span class="nam">W_O</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span><span class="nam">W_O</span><span class="op">,</span> <span class="str">"m (n h)->n h m"</span><span class="op">,</span> <span class="nam">n</span><span class="op">=</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_heads</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t58" href="#t58">58</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_O"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">W_O</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t56" href="#t56">56</a></span><span class="t">        <span class="nam">W_O</span> <span class="op">=</span> <span class="nam">qwen</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">layers</span><span class="op">[</span><span class="nam">l</span><span class="op">]</span><span class="op">.</span><span class="nam">self_attn</span><span class="op">.</span><span class="nam">o_proj</span><span class="op">.</span><span class="nam">weight</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t57" href="#t57">57</a></span><span class="t">        <span class="nam">W_O</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span><span class="nam">W_O</span><span class="op">,</span> <span class="str">"m (n h)->n h m"</span><span class="op">,</span> <span class="nam">n</span><span class="op">=</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_heads</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t58" href="#t58">58</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_O"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">W_O</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t59" href="#t59">59</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t60" href="#t60">60</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_O"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros</span><span class="op">(</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_model</span><span class="op">,</span> <span class="nam">dtype</span><span class="op">=</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">dtype</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t60" href="#t60">60</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_O"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros</span><span class="op">(</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_model</span><span class="op">,</span> <span class="nam">dtype</span><span class="op">=</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">dtype</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t61" href="#t61">61</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t62" href="#t62">62</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln2.w"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">qwen</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">layers</span><span class="op">[</span><span class="nam">l</span><span class="op">]</span><span class="op">.</span><span class="nam">post_attention_layernorm</span><span class="op">.</span><span class="nam">weight</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t62" href="#t62">62</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln2.w"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">qwen</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">layers</span><span class="op">[</span><span class="nam">l</span><span class="op">]</span><span class="op">.</span><span class="nam">post_attention_layernorm</span><span class="op">.</span><span class="nam">weight</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t63" href="#t63">63</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t64" href="#t64">64</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_in"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">qwen</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">layers</span><span class="op">[</span><span class="nam">l</span><span class="op">]</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">up_proj</span><span class="op">.</span><span class="nam">weight</span><span class="op">.</span><span class="nam">T</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t65" href="#t65">65</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_gate"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">qwen</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">layers</span><span class="op">[</span><span class="nam">l</span><span class="op">]</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">gate_proj</span><span class="op">.</span><span class="nam">weight</span><span class="op">.</span><span class="nam">T</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t66" href="#t66">66</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.b_in"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros</span><span class="op">(</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_mlp</span><span class="op">,</span> <span class="nam">dtype</span><span class="op">=</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">dtype</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t64" href="#t64">64</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_in"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">qwen</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">layers</span><span class="op">[</span><span class="nam">l</span><span class="op">]</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">up_proj</span><span class="op">.</span><span class="nam">weight</span><span class="op">.</span><span class="nam">T</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t65" href="#t65">65</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_gate"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">qwen</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">layers</span><span class="op">[</span><span class="nam">l</span><span class="op">]</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">gate_proj</span><span class="op">.</span><span class="nam">weight</span><span class="op">.</span><span class="nam">T</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t66" href="#t66">66</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.b_in"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros</span><span class="op">(</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_mlp</span><span class="op">,</span> <span class="nam">dtype</span><span class="op">=</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">dtype</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t67" href="#t67">67</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t68" href="#t68">68</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_out"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">qwen</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">layers</span><span class="op">[</span><span class="nam">l</span><span class="op">]</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">down_proj</span><span class="op">.</span><span class="nam">weight</span><span class="op">.</span><span class="nam">T</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t69" href="#t69">69</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.b_out"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros</span><span class="op">(</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_model</span><span class="op">,</span> <span class="nam">dtype</span><span class="op">=</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">dtype</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t68" href="#t68">68</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_out"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">qwen</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">layers</span><span class="op">[</span><span class="nam">l</span><span class="op">]</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">down_proj</span><span class="op">.</span><span class="nam">weight</span><span class="op">.</span><span class="nam">T</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t69" href="#t69">69</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.b_out"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros</span><span class="op">(</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_model</span><span class="op">,</span> <span class="nam">dtype</span><span class="op">=</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">dtype</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t70" href="#t70">70</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t71" href="#t71">71</a></span><span class="t">    <span class="nam">state_dict</span><span class="op">[</span><span class="str">"ln_final.w"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">qwen</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">norm</span><span class="op">.</span><span class="nam">weight</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t71" href="#t71">71</a></span><span class="t">    <span class="nam">state_dict</span><span class="op">[</span><span class="str">"ln_final.w"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">qwen</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">norm</span><span class="op">.</span><span class="nam">weight</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t72" href="#t72">72</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t73" href="#t73">73</a></span><span class="t">    <span class="nam">state_dict</span><span class="op">[</span><span class="str">"unembed.W_U"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">qwen</span><span class="op">.</span><span class="nam">lm_head</span><span class="op">.</span><span class="nam">weight</span><span class="op">.</span><span class="nam">T</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t74" href="#t74">74</a></span><span class="t">    <span class="nam">state_dict</span><span class="op">[</span><span class="str">"unembed.b_U"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros</span><span class="op">(</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_vocab</span><span class="op">,</span> <span class="nam">dtype</span><span class="op">=</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">dtype</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t73" href="#t73">73</a></span><span class="t">    <span class="nam">state_dict</span><span class="op">[</span><span class="str">"unembed.W_U"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">qwen</span><span class="op">.</span><span class="nam">lm_head</span><span class="op">.</span><span class="nam">weight</span><span class="op">.</span><span class="nam">T</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t74" href="#t74">74</a></span><span class="t">    <span class="nam">state_dict</span><span class="op">[</span><span class="str">"unembed.b_U"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros</span><span class="op">(</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_vocab</span><span class="op">,</span> <span class="nam">dtype</span><span class="op">=</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">dtype</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t75" href="#t75">75</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
     <p class="run"><span class="n"><a id="t76" href="#t76">76</a></span><span class="t">    <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
 </main>
@@ -167,7 +167,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_t5_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_qwen_py.html b/_static/coverage/d_c1ea89878f9b2ac7_qwen_py.html
index 45cde4ff1..0dcbdee5a 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_qwen_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_qwen_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_qwen2_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -156,7 +156,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_qwen2_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_t5_py.html b/_static/coverage/d_c1ea89878f9b2ac7_t5_py.html
index 27018b692..c26640c0a 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_t5_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_t5_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_train_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -192,7 +192,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_train_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831___init___py.html b/_static/coverage/d_db46118ef83ad831___init___py.html
index cd4e69703..7fdf450af 100644
--- a/_static/coverage/d_db46118ef83ad831___init___py.html
+++ b/_static/coverage/d_db46118ef83ad831___init___py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_abstract_attention_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -123,7 +123,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_abstract_attention_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_abstract_attention_py.html b/_static/coverage/d_db46118ef83ad831_abstract_attention_py.html
index 9fb97ef57..76bd1e9ff 100644
--- a/_static/coverage/d_db46118ef83ad831_abstract_attention_py.html
+++ b/_static/coverage/d_db46118ef83ad831_abstract_attention_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_attention_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -800,7 +800,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_attention_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_attention_py.html b/_static/coverage/d_db46118ef83ad831_attention_py.html
index 54cd8117a..9727e8fb4 100644
--- a/_static/coverage/d_db46118ef83ad831_attention_py.html
+++ b/_static/coverage/d_db46118ef83ad831_attention_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_bert_block_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -150,7 +150,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_bert_block_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_bert_block_py.html b/_static/coverage/d_db46118ef83ad831_bert_block_py.html
index 4061fbe61..ff834b7ba 100644
--- a/_static/coverage/d_db46118ef83ad831_bert_block_py.html
+++ b/_static/coverage/d_db46118ef83ad831_bert_block_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_bert_embed_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -168,7 +168,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_bert_embed_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_bert_embed_py.html b/_static/coverage/d_db46118ef83ad831_bert_embed_py.html
index 6b0ff4772..47a3c5e86 100644
--- a/_static/coverage/d_db46118ef83ad831_bert_embed_py.html
+++ b/_static/coverage/d_db46118ef83ad831_bert_embed_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_bert_mlm_head_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -143,7 +143,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_bert_mlm_head_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_bert_mlm_head_py.html b/_static/coverage/d_db46118ef83ad831_bert_mlm_head_py.html
index 39853e365..d4fbd0ddb 100644
--- a/_static/coverage/d_db46118ef83ad831_bert_mlm_head_py.html
+++ b/_static/coverage/d_db46118ef83ad831_bert_mlm_head_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_embed_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -132,7 +132,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_embed_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_embed_py.html b/_static/coverage/d_db46118ef83ad831_embed_py.html
index 767eb2482..dc88c260e 100644
--- a/_static/coverage/d_db46118ef83ad831_embed_py.html
+++ b/_static/coverage/d_db46118ef83ad831_embed_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_grouped_query_attention_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -125,7 +125,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_grouped_query_attention_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_grouped_query_attention_py.html b/_static/coverage/d_db46118ef83ad831_grouped_query_attention_py.html
index 95dd9b30c..813e0c7b2 100644
--- a/_static/coverage/d_db46118ef83ad831_grouped_query_attention_py.html
+++ b/_static/coverage/d_db46118ef83ad831_grouped_query_attention_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_layer_norm_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -269,7 +269,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_layer_norm_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_layer_norm_pre_py.html b/_static/coverage/d_db46118ef83ad831_layer_norm_pre_py.html
index 9a5257d55..c7a236e6e 100644
--- a/_static/coverage/d_db46118ef83ad831_layer_norm_pre_py.html
+++ b/_static/coverage/d_db46118ef83ad831_layer_norm_pre_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_can_be_used_as_mlp_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -144,7 +144,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_can_be_used_as_mlp_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_layer_norm_py.html b/_static/coverage/d_db46118ef83ad831_layer_norm_py.html
index d962c34ae..4c1656f57 100644
--- a/_static/coverage/d_db46118ef83ad831_layer_norm_py.html
+++ b/_static/coverage/d_db46118ef83ad831_layer_norm_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_layer_norm_pre_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -147,7 +147,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_layer_norm_pre_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_pos_embed_py.html b/_static/coverage/d_db46118ef83ad831_pos_embed_py.html
index 27f68afbb..0b7e7b3b4 100644
--- a/_static/coverage/d_db46118ef83ad831_pos_embed_py.html
+++ b/_static/coverage/d_db46118ef83ad831_pos_embed_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_rms_norm_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -160,7 +160,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_rms_norm_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_rms_norm_pre_py.html b/_static/coverage/d_db46118ef83ad831_rms_norm_pre_py.html
index 42aeccb4c..3d28f357f 100644
--- a/_static/coverage/d_db46118ef83ad831_rms_norm_pre_py.html
+++ b/_static/coverage/d_db46118ef83ad831_rms_norm_pre_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_t5_attention_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -126,7 +126,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_t5_attention_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_rms_norm_py.html b/_static/coverage/d_db46118ef83ad831_rms_norm_py.html
index 0cbd2d219..a1c0a49ad 100644
--- a/_static/coverage/d_db46118ef83ad831_rms_norm_py.html
+++ b/_static/coverage/d_db46118ef83ad831_rms_norm_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_rms_norm_pre_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -136,7 +136,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_rms_norm_pre_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_t5_attention_py.html b/_static/coverage/d_db46118ef83ad831_t5_attention_py.html
index 6693d77bf..1eb6564f1 100644
--- a/_static/coverage/d_db46118ef83ad831_t5_attention_py.html
+++ b/_static/coverage/d_db46118ef83ad831_t5_attention_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_t5_block_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -231,7 +231,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_t5_block_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_t5_block_py.html b/_static/coverage/d_db46118ef83ad831_t5_block_py.html
index 0f0d013f2..f9c307c38 100644
--- a/_static/coverage/d_db46118ef83ad831_t5_block_py.html
+++ b/_static/coverage/d_db46118ef83ad831_t5_block_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_token_typed_embed_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -247,7 +247,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_token_typed_embed_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_token_typed_embed_py.html b/_static/coverage/d_db46118ef83ad831_token_typed_embed_py.html
index f18fd7d5a..dccb1ef3b 100644
--- a/_static/coverage/d_db46118ef83ad831_token_typed_embed_py.html
+++ b/_static/coverage/d_db46118ef83ad831_token_typed_embed_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_transformer_block_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -118,7 +118,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_transformer_block_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_transformer_block_py.html b/_static/coverage/d_db46118ef83ad831_transformer_block_py.html
index 44030eb62..ad93832b0 100644
--- a/_static/coverage/d_db46118ef83ad831_transformer_block_py.html
+++ b/_static/coverage/d_db46118ef83ad831_transformer_block_py.html
@@ -2,7 +2,7 @@
 <html>
 <head>
     <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
-    <title>Coverage for transformer_lens/components/transformer_block.py: 78%</title>
+    <title>Coverage for transformer_lens/components/transformer_block.py: 80%</title>
     <link rel="icon" sizes="32x32" href="favicon_32.png">
     <link rel="stylesheet" href="style.css" type="text/css">
     <script type="text/javascript" src="coverage_html.js" defer></script>
@@ -12,7 +12,7 @@
     <div class="content">
         <h1>
             <span class="text">Coverage for </span><b>transformer_lens/components/transformer_block.py</b>:
-            <span class="pc_cov">78%</span>
+            <span class="pc_cov">80%</span>
         </h1>
         <aside id="help_panel_wrapper">
             <input id="help_panel_state" type="checkbox">
@@ -56,10 +56,10 @@ <h1>
         </aside>
         <h2>
             <span class="text">101 statements &nbsp;</span>
-            <button type="button" class="run button_toggle_run" value="run" data-shortcut="r" title="Toggle lines run">86<span class="text"> run</span></button>
-            <button type="button" class="mis show_mis button_toggle_mis" value="mis" data-shortcut="m" title="Toggle lines missing">15<span class="text"> missing</span></button>
+            <button type="button" class="run button_toggle_run" value="run" data-shortcut="r" title="Toggle lines run">87<span class="text"> run</span></button>
+            <button type="button" class="mis show_mis button_toggle_mis" value="mis" data-shortcut="m" title="Toggle lines missing">14<span class="text"> missing</span></button>
             <button type="button" class="exc show_exc button_toggle_exc" value="exc" data-shortcut="x" title="Toggle lines excluded">0<span class="text"> excluded</span></button>
-            <button type="button" class="par run show_par button_toggle_par" value="par" data-shortcut="p" title="Toggle lines partially run">10<span class="text"> partial</span></button>
+            <button type="button" class="par run show_par button_toggle_par" value="par" data-shortcut="p" title="Toggle lines partially run">9<span class="text"> partial</span></button>
         </h2>
         <p class="text">
             <a id="prevFileLink" class="nav" href="d_db46118ef83ad831_token_typed_embed_py.html">&#xab; prev</a> &nbsp; &nbsp;
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_unembed_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -130,8 +130,8 @@ <h2>
     <p class="run"><span class="n"><a id="t46" href="#t46">46</a></span><span class="t">            <span class="nam">normalization_layer</span> <span class="op">=</span> <span class="nam">LayerNormPre</span>&nbsp;</span><span class="r"></span></p>
     <p class="par run show_par"><span class="n"><a id="t47" href="#t47">47</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">==</span> <span class="str">"RMS"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">47&#x202F;&#x219B;&#x202F;48</span><span class="annotate long">line 47 didn't jump to line 48, because the condition on line 47 was never true</span></span></p>
     <p class="mis show_mis"><span class="n"><a id="t48" href="#t48">48</a></span><span class="t">            <span class="nam">normalization_layer</span> <span class="op">=</span> <span class="nam">RMSNorm</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t49" href="#t49">49</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">==</span> <span class="str">"RMSPre"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">49&#x202F;&#x219B;&#x202F;50</span><span class="annotate long">line 49 didn't jump to line 50, because the condition on line 49 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t50" href="#t50">50</a></span><span class="t">            <span class="nam">normalization_layer</span> <span class="op">=</span> <span class="nam">RMSNormPre</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t49" href="#t49">49</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">==</span> <span class="str">"RMSPre"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t50" href="#t50">50</a></span><span class="t">            <span class="nam">normalization_layer</span> <span class="op">=</span> <span class="nam">RMSNormPre</span>&nbsp;</span><span class="r"></span></p>
     <p class="par run show_par"><span class="n"><a id="t51" href="#t51">51</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">51&#x202F;&#x219B;&#x202F;56</span><span class="annotate long">line 51 didn't jump to line 56, because the condition on line 51 was never false</span></span></p>
     <p class="pln"><span class="n"><a id="t52" href="#t52">52</a></span><span class="t">            <span class="com"># This should just be the identity.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t53" href="#t53">53</a></span><span class="t">            <span class="com"># We need to make this a lambda so we can call it on the config, just like the others</span>&nbsp;</span><span class="r"></span></p>
@@ -300,7 +300,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_unembed_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_unembed_py.html b/_static/coverage/d_db46118ef83ad831_unembed_py.html
index aef4ffa26..60a92e71c 100644
--- a/_static/coverage/d_db46118ef83ad831_unembed_py.html
+++ b/_static/coverage/d_db46118ef83ad831_unembed_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_evals_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -122,7 +122,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_evals_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/index.html b/_static/coverage/index.html
index ad85dec08..3699025a9 100644
--- a/_static/coverage/index.html
+++ b/_static/coverage/index.html
@@ -11,7 +11,7 @@
 <header>
     <div class="content">
         <h1>Coverage report:
-            <span class="pc_cov">74%</span>
+            <span class="pc_cov">75%</span>
         </h1>
         <aside id="help_panel_wrapper">
             <input id="help_panel_state" type="checkbox">
@@ -47,7 +47,7 @@ <h1>Coverage report:
         </form>
         <p class="text">
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </header>
@@ -103,12 +103,12 @@ <h1>Coverage report:
             </tr>
             <tr class="file">
                 <td class="name left"><a href="d_af97b5493da09a14_HookedTransformer_py.html">transformer_lens/HookedTransformer.py</a></td>
-                <td>740</td>
-                <td>141</td>
+                <td>741</td>
+                <td>132</td>
                 <td>0</td>
                 <td>456</td>
-                <td>90</td>
-                <td class="right" data-ratio="897 1196">75%</td>
+                <td>79</td>
+                <td class="right" data-ratio="920 1197">77%</td>
             </tr>
             <tr class="file">
                 <td class="name left"><a href="d_af97b5493da09a14_HookedTransformerConfig_py.html">transformer_lens/HookedTransformerConfig.py</a></td>
@@ -116,8 +116,8 @@ <h1>Coverage report:
                 <td>8</td>
                 <td>0</td>
                 <td>40</td>
-                <td>7</td>
-                <td class="right" data-ratio="160 175">91%</td>
+                <td>6</td>
+                <td class="right" data-ratio="161 175">92%</td>
             </tr>
             <tr class="file">
                 <td class="name left"><a href="d_af97b5493da09a14_SVDInterpreter_py.html">transformer_lens/SVDInterpreter.py</a></td>
@@ -329,11 +329,11 @@ <h1>Coverage report:
             <tr class="file">
                 <td class="name left"><a href="d_db46118ef83ad831_transformer_block_py.html">transformer_lens/components/transformer_block.py</a></td>
                 <td>101</td>
-                <td>15</td>
+                <td>14</td>
                 <td>0</td>
                 <td>52</td>
-                <td>10</td>
-                <td class="right" data-ratio="120 153">78%</td>
+                <td>9</td>
+                <td class="right" data-ratio="122 153">80%</td>
             </tr>
             <tr class="file">
                 <td class="name left"><a href="d_db46118ef83ad831_unembed_py.html">transformer_lens/components/unembed.py</a></td>
@@ -391,12 +391,12 @@ <h1>Coverage report:
             </tr>
             <tr class="file">
                 <td class="name left"><a href="d_af97b5493da09a14_loading_from_pretrained_py.html">transformer_lens/loading_from_pretrained.py</a></td>
-                <td>320</td>
-                <td>112</td>
+                <td>322</td>
+                <td>106</td>
                 <td>0</td>
-                <td>216</td>
-                <td>51</td>
-                <td class="right" data-ratio="331 536">62%</td>
+                <td>218</td>
+                <td>53</td>
+                <td class="right" data-ratio="347 540">64%</td>
             </tr>
             <tr class="file">
                 <td class="name left"><a href="d_af97b5493da09a14_past_key_value_caching_py.html">transformer_lens/past_key_value_caching.py</a></td>
@@ -599,11 +599,11 @@ <h1>Coverage report:
             <tr class="file">
                 <td class="name left"><a href="d_c1ea89878f9b2ac7_qwen2_py.html">transformer_lens/pretrained/weight_conversions/qwen2.py</a></td>
                 <td>41</td>
-                <td>36</td>
+                <td>0</td>
                 <td>0</td>
                 <td>2</td>
                 <td>0</td>
-                <td class="right" data-ratio="5 43">12%</td>
+                <td class="right" data-ratio="43 43">100%</td>
             </tr>
             <tr class="file">
                 <td class="name left"><a href="d_c1ea89878f9b2ac7_t5_py.html">transformer_lens/pretrained/weight_conversions/t5.py</a></td>
@@ -681,12 +681,12 @@ <h1>Coverage report:
         <tfoot>
             <tr class="total">
                 <td class="name left">Total</td>
-                <td>5038</td>
-                <td>1139</td>
+                <td>5041</td>
+                <td>1087</td>
                 <td>0</td>
-                <td>1986</td>
-                <td>336</td>
-                <td class="right" data-ratio="5207 7024">74%</td>
+                <td>1988</td>
+                <td>325</td>
+                <td class="right" data-ratio="5287 7029">75%</td>
             </tr>
         </tfoot>
     </table>
@@ -698,7 +698,7 @@ <h1>Coverage report:
     <div class="content">
         <p>
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
     <aside class="hidden">
diff --git a/_static/coverage/status.json b/_static/coverage/status.json
index 976126417..673b1cc6f 100644
--- a/_static/coverage/status.json
+++ b/_static/coverage/status.json
@@ -1 +1 @@
-{"note":"This file is an internal implementation detail to speed up HTML report generation. Its format can change at any time. You might be looking for the JSON report: https://coverage.rtfd.io/cmd.html#cmd-json","format":2,"version":"7.4.4","globals":"f7ec83bc01aec1de67db4e1d62b20997","files":{"d_af97b5493da09a14_ActivationCache_py":{"hash":"5ce55533b35bb3aea32c87d3444573f7","index":{"nums":[0,1,289,0,7,168,10,16],"html_filename":"d_af97b5493da09a14_ActivationCache_py.html","relative_filename":"transformer_lens/ActivationCache.py"}},"d_af97b5493da09a14_FactoredMatrix_py":{"hash":"3731c82dd9da1b8aa7d998cf8cf5db55","index":{"nums":[0,1,129,0,2,60,3,5],"html_filename":"d_af97b5493da09a14_FactoredMatrix_py.html","relative_filename":"transformer_lens/FactoredMatrix.py"}},"d_af97b5493da09a14_HookedEncoder_py":{"hash":"ecf3200c538017a5f4a2a6d8a4bf9c4f","index":{"nums":[0,1,162,0,10,104,33,35],"html_filename":"d_af97b5493da09a14_HookedEncoder_py.html","relative_filename":"transformer_lens/HookedEncoder.py"}},"d_af97b5493da09a14_HookedEncoderDecoder_py":{"hash":"ced41e685cd841a10cfabecef98df15c","index":{"nums":[0,1,170,0,19,112,41,45],"html_filename":"d_af97b5493da09a14_HookedEncoderDecoder_py.html","relative_filename":"transformer_lens/HookedEncoderDecoder.py"}},"d_af97b5493da09a14_HookedTransformer_py":{"hash":"894cd039c08fbb4b4023464453b62b01","index":{"nums":[0,1,740,0,141,456,90,158],"html_filename":"d_af97b5493da09a14_HookedTransformer_py.html","relative_filename":"transformer_lens/HookedTransformer.py"}},"d_af97b5493da09a14_HookedTransformerConfig_py":{"hash":"54ef75dda88a6bf77428fc230a984c66","index":{"nums":[0,1,135,0,8,40,7,7],"html_filename":"d_af97b5493da09a14_HookedTransformerConfig_py.html","relative_filename":"transformer_lens/HookedTransformerConfig.py"}},"d_af97b5493da09a14_SVDInterpreter_py":{"hash":"16a2ab71f04385f7b023d23a2bdef6d8","index":{"nums":[0,1,54,0,1,16,1,1],"html_filename":"d_af97b5493da09a14_SVDInterpreter_py.html","relative_filename":"transformer_lens/SVDInterpreter.py"}},"d_af97b5493da09a14___init___py":{"hash":"b5c172d6f5d0c163dd8d3d8a188ef8fc","index":{"nums":[0,1,20,0,0,0,0,0],"html_filename":"d_af97b5493da09a14___init___py.html","relative_filename":"transformer_lens/__init__.py"}},"d_db46118ef83ad831___init___py":{"hash":"0f5ac43371106089cb8859730506eb31","index":{"nums":[0,1,20,0,0,0,0,0],"html_filename":"d_db46118ef83ad831___init___py.html","relative_filename":"transformer_lens/components/__init__.py"}},"d_db46118ef83ad831_abstract_attention_py":{"hash":"cfb43d36db80beb3b8f25c159abad3cc","index":{"nums":[0,1,240,0,41,86,19,23],"html_filename":"d_db46118ef83ad831_abstract_attention_py.html","relative_filename":"transformer_lens/components/abstract_attention.py"}},"d_db46118ef83ad831_attention_py":{"hash":"771ce2a06e2204a8ea0b5c09549b0faa","index":{"nums":[0,1,20,0,4,4,2,2],"html_filename":"d_db46118ef83ad831_attention_py.html","relative_filename":"transformer_lens/components/attention.py"}},"d_db46118ef83ad831_bert_block_py":{"hash":"1d3faf4dfb51c82ef0e70e027c4442f7","index":{"nums":[0,1,45,0,4,2,1,1],"html_filename":"d_db46118ef83ad831_bert_block_py.html","relative_filename":"transformer_lens/components/bert_block.py"}},"d_db46118ef83ad831_bert_embed_py":{"hash":"fbb9b2fc553abf0e4d8a0afa34f58817","index":{"nums":[0,1,30,0,0,2,0,0],"html_filename":"d_db46118ef83ad831_bert_embed_py.html","relative_filename":"transformer_lens/components/bert_embed.py"}},"d_db46118ef83ad831_bert_mlm_head_py":{"hash":"13e34c2e7e8b18edf5fd230ccec94ead","index":{"nums":[0,1,22,0,0,0,0,0],"html_filename":"d_db46118ef83ad831_bert_mlm_head_py.html","relative_filename":"transformer_lens/components/bert_mlm_head.py"}},"d_db46118ef83ad831_embed_py":{"hash":"e80917c423b62b1fffeee5455ec2d952","index":{"nums":[0,1,17,0,0,4,0,0],"html_filename":"d_db46118ef83ad831_embed_py.html","relative_filename":"transformer_lens/components/embed.py"}},"d_db46118ef83ad831_grouped_query_attention_py":{"hash":"52d623d476218710d87c4c7081b05665","index":{"nums":[0,1,55,0,0,20,0,0],"html_filename":"d_db46118ef83ad831_grouped_query_attention_py.html","relative_filename":"transformer_lens/components/grouped_query_attention.py"}},"d_db46118ef83ad831_layer_norm_py":{"hash":"5d34676f9227c3c3bd83d13ca4a762ec","index":{"nums":[0,1,25,0,1,4,1,1],"html_filename":"d_db46118ef83ad831_layer_norm_py.html","relative_filename":"transformer_lens/components/layer_norm.py"}},"d_db46118ef83ad831_layer_norm_pre_py":{"hash":"a488e1ab3adf495070bf5e03582eee99","index":{"nums":[0,1,19,0,1,2,1,1],"html_filename":"d_db46118ef83ad831_layer_norm_pre_py.html","relative_filename":"transformer_lens/components/layer_norm_pre.py"}},"d_37285d613390727b_can_be_used_as_mlp_py":{"hash":"199b3e1e1d8fc5533d79c3788e257832","index":{"nums":[0,1,30,0,1,6,1,1],"html_filename":"d_37285d613390727b_can_be_used_as_mlp_py.html","relative_filename":"transformer_lens/components/mlps/can_be_used_as_mlp.py"}},"d_37285d613390727b_gated_mlp_py":{"hash":"1b9d23e893279e5255ac14d69e026485","index":{"nums":[0,1,31,0,3,4,2,2],"html_filename":"d_37285d613390727b_gated_mlp_py.html","relative_filename":"transformer_lens/components/mlps/gated_mlp.py"}},"d_37285d613390727b_gated_mlp_4bit_py":{"hash":"e85f0742f0a4142be44d967799763a15","index":{"nums":[0,1,32,0,18,4,1,3],"html_filename":"d_37285d613390727b_gated_mlp_4bit_py.html","relative_filename":"transformer_lens/components/mlps/gated_mlp_4bit.py"}},"d_37285d613390727b_mlp_py":{"hash":"a3901498f90188ad75b2b640144e7070","index":{"nums":[0,1,25,0,0,2,0,0],"html_filename":"d_37285d613390727b_mlp_py.html","relative_filename":"transformer_lens/components/mlps/mlp.py"}},"d_37285d613390727b_moe_py":{"hash":"c69707a0407a7cacf0929fa3233216f9","index":{"nums":[0,1,59,0,1,6,1,1],"html_filename":"d_37285d613390727b_moe_py.html","relative_filename":"transformer_lens/components/mlps/moe.py"}},"d_db46118ef83ad831_pos_embed_py":{"hash":"3688508c2257f2f12a9e431f92fdcd2a","index":{"nums":[0,1,23,0,0,2,0,0],"html_filename":"d_db46118ef83ad831_pos_embed_py.html","relative_filename":"transformer_lens/components/pos_embed.py"}},"d_db46118ef83ad831_rms_norm_py":{"hash":"19408e8189b8bb5d095ae6bb05045ef1","index":{"nums":[0,1,23,0,2,4,2,2],"html_filename":"d_db46118ef83ad831_rms_norm_py.html","relative_filename":"transformer_lens/components/rms_norm.py"}},"d_db46118ef83ad831_rms_norm_pre_py":{"hash":"04bea994792415493e41118c6e6c4da2","index":{"nums":[0,1,18,0,1,2,1,1],"html_filename":"d_db46118ef83ad831_rms_norm_pre_py.html","relative_filename":"transformer_lens/components/rms_norm_pre.py"}},"d_db46118ef83ad831_t5_attention_py":{"hash":"b02b9250eaf1c84e129e8763676c2d48","index":{"nums":[0,1,50,0,3,12,3,3],"html_filename":"d_db46118ef83ad831_t5_attention_py.html","relative_filename":"transformer_lens/components/t5_attention.py"}},"d_db46118ef83ad831_t5_block_py":{"hash":"8efa9a61311e44c6263d8dd61588fac4","index":{"nums":[0,1,64,0,6,12,3,3],"html_filename":"d_db46118ef83ad831_t5_block_py.html","relative_filename":"transformer_lens/components/t5_block.py"}},"d_db46118ef83ad831_token_typed_embed_py":{"hash":"0265390bcc2c91e19e04aee0cc722a2b","index":{"nums":[0,1,12,0,0,0,0,0],"html_filename":"d_db46118ef83ad831_token_typed_embed_py.html","relative_filename":"transformer_lens/components/token_typed_embed.py"}},"d_db46118ef83ad831_transformer_block_py":{"hash":"521e7e01284a7150605bf1dae5d7d9f1","index":{"nums":[0,1,101,0,15,52,10,18],"html_filename":"d_db46118ef83ad831_transformer_block_py.html","relative_filename":"transformer_lens/components/transformer_block.py"}},"d_db46118ef83ad831_unembed_py":{"hash":"a662f40d2a2543805f0b9f0d06f5b5c4","index":{"nums":[0,1,14,0,0,0,0,0],"html_filename":"d_db46118ef83ad831_unembed_py.html","relative_filename":"transformer_lens/components/unembed.py"}},"d_af97b5493da09a14_evals_py":{"hash":"7d496aae736b2525c13fe25d5363328b","index":{"nums":[0,1,147,0,50,46,2,14],"html_filename":"d_af97b5493da09a14_evals_py.html","relative_filename":"transformer_lens/evals.py"}},"d_65d4430f90bfb219_activation_function_factory_py":{"hash":"c4701f74bdab03c4b2cf3f7cd8afd980","index":{"nums":[0,1,12,0,2,6,2,2],"html_filename":"d_65d4430f90bfb219_activation_function_factory_py.html","relative_filename":"transformer_lens/factories/activation_function_factory.py"}},"d_65d4430f90bfb219_mlp_factory_py":{"hash":"4822436b1bacb210c29f36d16e51b216","index":{"nums":[0,1,14,0,1,6,1,1],"html_filename":"d_65d4430f90bfb219_mlp_factory_py.html","relative_filename":"transformer_lens/factories/mlp_factory.py"}},"d_af97b5493da09a14_head_detector_py":{"hash":"373e4b6b6945f7eec6de3b6987c4a9de","index":{"nums":[0,1,84,0,2,32,2,2],"html_filename":"d_af97b5493da09a14_head_detector_py.html","relative_filename":"transformer_lens/head_detector.py"}},"d_af97b5493da09a14_hook_points_py":{"hash":"60d1e6604c991526f1d8a3906b8fd1c8","index":{"nums":[0,1,234,0,39,124,18,46],"html_filename":"d_af97b5493da09a14_hook_points_py.html","relative_filename":"transformer_lens/hook_points.py"}},"d_af97b5493da09a14_loading_from_pretrained_py":{"hash":"2873ff135731d2dd591a750cc83eb464","index":{"nums":[0,1,320,0,112,216,51,93],"html_filename":"d_af97b5493da09a14_loading_from_pretrained_py.html","relative_filename":"transformer_lens/loading_from_pretrained.py"}},"d_af97b5493da09a14_past_key_value_caching_py":{"hash":"994a13780a7f92b6a2ab53a37e662c15","index":{"nums":[0,1,46,0,0,18,2,2],"html_filename":"d_af97b5493da09a14_past_key_value_caching_py.html","relative_filename":"transformer_lens/past_key_value_caching.py"}},"d_af97b5493da09a14_patching_py":{"hash":"d96c96e4ecf5fcfe45d2838d27a9d918","index":{"nums":[0,1,140,0,70,18,0,14],"html_filename":"d_af97b5493da09a14_patching_py.html","relative_filename":"transformer_lens/patching.py"}},"d_712808f24eb400fe___init___py":{"hash":"ca2411bb2d5283cba14da3ce60f5feec","index":{"nums":[0,1,0,0,0,0,0,0],"html_filename":"d_712808f24eb400fe___init___py.html","relative_filename":"transformer_lens/pretrained/__init__.py"}},"d_c1ea89878f9b2ac7___init___py":{"hash":"2994f6b388ff62b0d84269971b0766e3","index":{"nums":[0,1,20,0,0,0,0,0],"html_filename":"d_c1ea89878f9b2ac7___init___py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/__init__.py"}},"d_c1ea89878f9b2ac7_bert_py":{"hash":"1de2f9e5bcd80d6bcc6794f7073bbe66","index":{"nums":[0,1,31,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_bert_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/bert.py"}},"d_c1ea89878f9b2ac7_bloom_py":{"hash":"28eee61cd9581bbb556c20abd44717d1","index":{"nums":[0,1,40,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_bloom_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/bloom.py"}},"d_c1ea89878f9b2ac7_coder_py":{"hash":"008f8c07ead34001e59d834e9672b208","index":{"nums":[0,1,43,0,38,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_coder_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/coder.py"}},"d_c1ea89878f9b2ac7_gemma_py":{"hash":"b3ee2563edaf1aec4f1db3777cf34c84","index":{"nums":[0,1,41,0,36,6,0,6],"html_filename":"d_c1ea89878f9b2ac7_gemma_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/gemma.py"}},"d_c1ea89878f9b2ac7_gpt2_py":{"hash":"b6864d10cf3b70f76be00e47695dbec2","index":{"nums":[0,1,39,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_gpt2_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/gpt2.py"}},"d_c1ea89878f9b2ac7_gptj_py":{"hash":"6b6d95ba6133fe6935f33c2431cec376","index":{"nums":[0,1,36,0,31,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_gptj_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/gptj.py"}},"d_c1ea89878f9b2ac7_llama_py":{"hash":"b283de32d58e72f23d1dde8b2a54f278","index":{"nums":[0,1,45,0,39,8,0,8],"html_filename":"d_c1ea89878f9b2ac7_llama_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/llama.py"}},"d_c1ea89878f9b2ac7_mingpt_py":{"hash":"8afdb1ccd308a76b7af9ea71404ed112","index":{"nums":[0,1,40,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_mingpt_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/mingpt.py"}},"d_c1ea89878f9b2ac7_mistral_py":{"hash":"f08ee3cdaa7c2dfb152ef82a4917497f","index":{"nums":[0,1,36,0,31,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_mistral_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/mistral.py"}},"d_c1ea89878f9b2ac7_mixtral_py":{"hash":"bb26007d26637b02b3869d4733d0d6e8","index":{"nums":[0,1,37,0,32,4,0,4],"html_filename":"d_c1ea89878f9b2ac7_mixtral_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/mixtral.py"}},"d_c1ea89878f9b2ac7_nanogpt_py":{"hash":"e2ff3818963256534c9ca6a4ff6566a5","index":{"nums":[0,1,52,0,47,10,0,10],"html_filename":"d_c1ea89878f9b2ac7_nanogpt_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/nanogpt.py"}},"d_c1ea89878f9b2ac7_neel_solu_old_py":{"hash":"d96be68dd2fba5dbcba015e7f8b10fc1","index":{"nums":[0,1,17,0,0,12,2,2],"html_filename":"d_c1ea89878f9b2ac7_neel_solu_old_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/neel_solu_old.py"}},"d_c1ea89878f9b2ac7_neo_py":{"hash":"67f2d6bb1adda03d8d3eacbe5e1a0cb4","index":{"nums":[0,1,37,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_neo_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/neo.py"}},"d_c1ea89878f9b2ac7_neox_py":{"hash":"fde01ee4309dd58bf3a6793ca3421f9e","index":{"nums":[0,1,34,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_neox_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/neox.py"}},"d_c1ea89878f9b2ac7_opt_py":{"hash":"5bd938981ffdf863cddf1e54a3526615","index":{"nums":[0,1,40,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_opt_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/opt.py"}},"d_c1ea89878f9b2ac7_phi_py":{"hash":"487a556d6755479e53397cfa39646490","index":{"nums":[0,1,41,0,37,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_phi_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/phi.py"}},"d_c1ea89878f9b2ac7_phi3_py":{"hash":"9d833a74a1082fc28aa204b928511d91","index":{"nums":[0,1,35,0,30,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_phi3_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/phi3.py"}},"d_c1ea89878f9b2ac7_qwen_py":{"hash":"5abcfa29a1d0456fb304a4b3b7ee2063","index":{"nums":[0,1,38,0,33,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_qwen_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/qwen.py"}},"d_c1ea89878f9b2ac7_qwen2_py":{"hash":"4f2b989be42046944eb76e787f12f8a4","index":{"nums":[0,1,41,0,36,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_qwen2_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/qwen2.py"}},"d_c1ea89878f9b2ac7_t5_py":{"hash":"fb47f0cb69cfc6778072b400fe6e53ef","index":{"nums":[0,1,33,0,0,4,0,0],"html_filename":"d_c1ea89878f9b2ac7_t5_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/t5.py"}},"d_af97b5493da09a14_train_py":{"hash":"74c4dc0140e6c628f623287b84708fb2","index":{"nums":[0,1,72,0,42,34,1,33],"html_filename":"d_af97b5493da09a14_train_py.html","relative_filename":"transformer_lens/train.py"}},"d_b2114f845e0399b7___init___py":{"hash":"ca2411bb2d5283cba14da3ce60f5feec","index":{"nums":[0,1,0,0,0,0,0,0],"html_filename":"d_b2114f845e0399b7___init___py.html","relative_filename":"transformer_lens/utilities/__init__.py"}},"d_b2114f845e0399b7_activation_functions_py":{"hash":"0926a004863506fd686e3d1e0ae5eb84","index":{"nums":[0,1,6,0,0,2,0,0],"html_filename":"d_b2114f845e0399b7_activation_functions_py.html","relative_filename":"transformer_lens/utilities/activation_functions.py"}},"d_b2114f845e0399b7_addmm_py":{"hash":"b1cb8300653ca4c3dd6a4f5c06408a84","index":{"nums":[0,1,10,0,0,0,0,0],"html_filename":"d_b2114f845e0399b7_addmm_py.html","relative_filename":"transformer_lens/utilities/addmm.py"}},"d_b2114f845e0399b7_attention_py":{"hash":"a1d30d709e8539cc4515e76ad414a3ae","index":{"nums":[0,1,14,0,0,0,0,0],"html_filename":"d_b2114f845e0399b7_attention_py.html","relative_filename":"transformer_lens/utilities/attention.py"}},"d_b2114f845e0399b7_devices_py":{"hash":"1da1c1a01ae3a7697bbb04016881df11","index":{"nums":[0,1,31,0,7,18,4,10],"html_filename":"d_b2114f845e0399b7_devices_py.html","relative_filename":"transformer_lens/utilities/devices.py"}},"d_af97b5493da09a14_utils_py":{"hash":"d2b4d5cb90c365cbb3ef3981a81e5ae9","index":{"nums":[0,1,458,0,135,208,18,88],"html_filename":"d_af97b5493da09a14_utils_py.html","relative_filename":"transformer_lens/utils.py"}}}}
\ No newline at end of file
+{"note":"This file is an internal implementation detail to speed up HTML report generation. Its format can change at any time. You might be looking for the JSON report: https://coverage.rtfd.io/cmd.html#cmd-json","format":2,"version":"7.4.4","globals":"f7ec83bc01aec1de67db4e1d62b20997","files":{"d_af97b5493da09a14_ActivationCache_py":{"hash":"5ce55533b35bb3aea32c87d3444573f7","index":{"nums":[0,1,289,0,7,168,10,16],"html_filename":"d_af97b5493da09a14_ActivationCache_py.html","relative_filename":"transformer_lens/ActivationCache.py"}},"d_af97b5493da09a14_FactoredMatrix_py":{"hash":"3731c82dd9da1b8aa7d998cf8cf5db55","index":{"nums":[0,1,129,0,2,60,3,5],"html_filename":"d_af97b5493da09a14_FactoredMatrix_py.html","relative_filename":"transformer_lens/FactoredMatrix.py"}},"d_af97b5493da09a14_HookedEncoder_py":{"hash":"ecf3200c538017a5f4a2a6d8a4bf9c4f","index":{"nums":[0,1,162,0,10,104,33,35],"html_filename":"d_af97b5493da09a14_HookedEncoder_py.html","relative_filename":"transformer_lens/HookedEncoder.py"}},"d_af97b5493da09a14_HookedEncoderDecoder_py":{"hash":"ced41e685cd841a10cfabecef98df15c","index":{"nums":[0,1,170,0,19,112,41,45],"html_filename":"d_af97b5493da09a14_HookedEncoderDecoder_py.html","relative_filename":"transformer_lens/HookedEncoderDecoder.py"}},"d_af97b5493da09a14_HookedTransformer_py":{"hash":"4a6be10862df8f2c54b7be48ae6e26b7","index":{"nums":[0,1,741,0,132,456,79,145],"html_filename":"d_af97b5493da09a14_HookedTransformer_py.html","relative_filename":"transformer_lens/HookedTransformer.py"}},"d_af97b5493da09a14_HookedTransformerConfig_py":{"hash":"c836c9729cd8ce8b03fa1d8243cc1175","index":{"nums":[0,1,135,0,8,40,6,6],"html_filename":"d_af97b5493da09a14_HookedTransformerConfig_py.html","relative_filename":"transformer_lens/HookedTransformerConfig.py"}},"d_af97b5493da09a14_SVDInterpreter_py":{"hash":"16a2ab71f04385f7b023d23a2bdef6d8","index":{"nums":[0,1,54,0,1,16,1,1],"html_filename":"d_af97b5493da09a14_SVDInterpreter_py.html","relative_filename":"transformer_lens/SVDInterpreter.py"}},"d_af97b5493da09a14___init___py":{"hash":"b5c172d6f5d0c163dd8d3d8a188ef8fc","index":{"nums":[0,1,20,0,0,0,0,0],"html_filename":"d_af97b5493da09a14___init___py.html","relative_filename":"transformer_lens/__init__.py"}},"d_db46118ef83ad831___init___py":{"hash":"0f5ac43371106089cb8859730506eb31","index":{"nums":[0,1,20,0,0,0,0,0],"html_filename":"d_db46118ef83ad831___init___py.html","relative_filename":"transformer_lens/components/__init__.py"}},"d_db46118ef83ad831_abstract_attention_py":{"hash":"cfb43d36db80beb3b8f25c159abad3cc","index":{"nums":[0,1,240,0,41,86,19,23],"html_filename":"d_db46118ef83ad831_abstract_attention_py.html","relative_filename":"transformer_lens/components/abstract_attention.py"}},"d_db46118ef83ad831_attention_py":{"hash":"771ce2a06e2204a8ea0b5c09549b0faa","index":{"nums":[0,1,20,0,4,4,2,2],"html_filename":"d_db46118ef83ad831_attention_py.html","relative_filename":"transformer_lens/components/attention.py"}},"d_db46118ef83ad831_bert_block_py":{"hash":"1d3faf4dfb51c82ef0e70e027c4442f7","index":{"nums":[0,1,45,0,4,2,1,1],"html_filename":"d_db46118ef83ad831_bert_block_py.html","relative_filename":"transformer_lens/components/bert_block.py"}},"d_db46118ef83ad831_bert_embed_py":{"hash":"fbb9b2fc553abf0e4d8a0afa34f58817","index":{"nums":[0,1,30,0,0,2,0,0],"html_filename":"d_db46118ef83ad831_bert_embed_py.html","relative_filename":"transformer_lens/components/bert_embed.py"}},"d_db46118ef83ad831_bert_mlm_head_py":{"hash":"13e34c2e7e8b18edf5fd230ccec94ead","index":{"nums":[0,1,22,0,0,0,0,0],"html_filename":"d_db46118ef83ad831_bert_mlm_head_py.html","relative_filename":"transformer_lens/components/bert_mlm_head.py"}},"d_db46118ef83ad831_embed_py":{"hash":"e80917c423b62b1fffeee5455ec2d952","index":{"nums":[0,1,17,0,0,4,0,0],"html_filename":"d_db46118ef83ad831_embed_py.html","relative_filename":"transformer_lens/components/embed.py"}},"d_db46118ef83ad831_grouped_query_attention_py":{"hash":"52d623d476218710d87c4c7081b05665","index":{"nums":[0,1,55,0,0,20,0,0],"html_filename":"d_db46118ef83ad831_grouped_query_attention_py.html","relative_filename":"transformer_lens/components/grouped_query_attention.py"}},"d_db46118ef83ad831_layer_norm_py":{"hash":"5d34676f9227c3c3bd83d13ca4a762ec","index":{"nums":[0,1,25,0,1,4,1,1],"html_filename":"d_db46118ef83ad831_layer_norm_py.html","relative_filename":"transformer_lens/components/layer_norm.py"}},"d_db46118ef83ad831_layer_norm_pre_py":{"hash":"a488e1ab3adf495070bf5e03582eee99","index":{"nums":[0,1,19,0,1,2,1,1],"html_filename":"d_db46118ef83ad831_layer_norm_pre_py.html","relative_filename":"transformer_lens/components/layer_norm_pre.py"}},"d_37285d613390727b_can_be_used_as_mlp_py":{"hash":"199b3e1e1d8fc5533d79c3788e257832","index":{"nums":[0,1,30,0,1,6,1,1],"html_filename":"d_37285d613390727b_can_be_used_as_mlp_py.html","relative_filename":"transformer_lens/components/mlps/can_be_used_as_mlp.py"}},"d_37285d613390727b_gated_mlp_py":{"hash":"1b9d23e893279e5255ac14d69e026485","index":{"nums":[0,1,31,0,3,4,2,2],"html_filename":"d_37285d613390727b_gated_mlp_py.html","relative_filename":"transformer_lens/components/mlps/gated_mlp.py"}},"d_37285d613390727b_gated_mlp_4bit_py":{"hash":"e85f0742f0a4142be44d967799763a15","index":{"nums":[0,1,32,0,18,4,1,3],"html_filename":"d_37285d613390727b_gated_mlp_4bit_py.html","relative_filename":"transformer_lens/components/mlps/gated_mlp_4bit.py"}},"d_37285d613390727b_mlp_py":{"hash":"a3901498f90188ad75b2b640144e7070","index":{"nums":[0,1,25,0,0,2,0,0],"html_filename":"d_37285d613390727b_mlp_py.html","relative_filename":"transformer_lens/components/mlps/mlp.py"}},"d_37285d613390727b_moe_py":{"hash":"c69707a0407a7cacf0929fa3233216f9","index":{"nums":[0,1,59,0,1,6,1,1],"html_filename":"d_37285d613390727b_moe_py.html","relative_filename":"transformer_lens/components/mlps/moe.py"}},"d_db46118ef83ad831_pos_embed_py":{"hash":"3688508c2257f2f12a9e431f92fdcd2a","index":{"nums":[0,1,23,0,0,2,0,0],"html_filename":"d_db46118ef83ad831_pos_embed_py.html","relative_filename":"transformer_lens/components/pos_embed.py"}},"d_db46118ef83ad831_rms_norm_py":{"hash":"19408e8189b8bb5d095ae6bb05045ef1","index":{"nums":[0,1,23,0,2,4,2,2],"html_filename":"d_db46118ef83ad831_rms_norm_py.html","relative_filename":"transformer_lens/components/rms_norm.py"}},"d_db46118ef83ad831_rms_norm_pre_py":{"hash":"04bea994792415493e41118c6e6c4da2","index":{"nums":[0,1,18,0,1,2,1,1],"html_filename":"d_db46118ef83ad831_rms_norm_pre_py.html","relative_filename":"transformer_lens/components/rms_norm_pre.py"}},"d_db46118ef83ad831_t5_attention_py":{"hash":"b02b9250eaf1c84e129e8763676c2d48","index":{"nums":[0,1,50,0,3,12,3,3],"html_filename":"d_db46118ef83ad831_t5_attention_py.html","relative_filename":"transformer_lens/components/t5_attention.py"}},"d_db46118ef83ad831_t5_block_py":{"hash":"8efa9a61311e44c6263d8dd61588fac4","index":{"nums":[0,1,64,0,6,12,3,3],"html_filename":"d_db46118ef83ad831_t5_block_py.html","relative_filename":"transformer_lens/components/t5_block.py"}},"d_db46118ef83ad831_token_typed_embed_py":{"hash":"0265390bcc2c91e19e04aee0cc722a2b","index":{"nums":[0,1,12,0,0,0,0,0],"html_filename":"d_db46118ef83ad831_token_typed_embed_py.html","relative_filename":"transformer_lens/components/token_typed_embed.py"}},"d_db46118ef83ad831_transformer_block_py":{"hash":"a18804da0ff942915f353adcf2f1ee58","index":{"nums":[0,1,101,0,14,52,9,17],"html_filename":"d_db46118ef83ad831_transformer_block_py.html","relative_filename":"transformer_lens/components/transformer_block.py"}},"d_db46118ef83ad831_unembed_py":{"hash":"a662f40d2a2543805f0b9f0d06f5b5c4","index":{"nums":[0,1,14,0,0,0,0,0],"html_filename":"d_db46118ef83ad831_unembed_py.html","relative_filename":"transformer_lens/components/unembed.py"}},"d_af97b5493da09a14_evals_py":{"hash":"7d496aae736b2525c13fe25d5363328b","index":{"nums":[0,1,147,0,50,46,2,14],"html_filename":"d_af97b5493da09a14_evals_py.html","relative_filename":"transformer_lens/evals.py"}},"d_65d4430f90bfb219_activation_function_factory_py":{"hash":"c4701f74bdab03c4b2cf3f7cd8afd980","index":{"nums":[0,1,12,0,2,6,2,2],"html_filename":"d_65d4430f90bfb219_activation_function_factory_py.html","relative_filename":"transformer_lens/factories/activation_function_factory.py"}},"d_65d4430f90bfb219_mlp_factory_py":{"hash":"4822436b1bacb210c29f36d16e51b216","index":{"nums":[0,1,14,0,1,6,1,1],"html_filename":"d_65d4430f90bfb219_mlp_factory_py.html","relative_filename":"transformer_lens/factories/mlp_factory.py"}},"d_af97b5493da09a14_head_detector_py":{"hash":"373e4b6b6945f7eec6de3b6987c4a9de","index":{"nums":[0,1,84,0,2,32,2,2],"html_filename":"d_af97b5493da09a14_head_detector_py.html","relative_filename":"transformer_lens/head_detector.py"}},"d_af97b5493da09a14_hook_points_py":{"hash":"60d1e6604c991526f1d8a3906b8fd1c8","index":{"nums":[0,1,234,0,39,124,18,46],"html_filename":"d_af97b5493da09a14_hook_points_py.html","relative_filename":"transformer_lens/hook_points.py"}},"d_af97b5493da09a14_loading_from_pretrained_py":{"hash":"0020574ad55675bf32c5f06e2cef02c5","index":{"nums":[0,1,322,0,106,218,53,87],"html_filename":"d_af97b5493da09a14_loading_from_pretrained_py.html","relative_filename":"transformer_lens/loading_from_pretrained.py"}},"d_af97b5493da09a14_past_key_value_caching_py":{"hash":"994a13780a7f92b6a2ab53a37e662c15","index":{"nums":[0,1,46,0,0,18,2,2],"html_filename":"d_af97b5493da09a14_past_key_value_caching_py.html","relative_filename":"transformer_lens/past_key_value_caching.py"}},"d_af97b5493da09a14_patching_py":{"hash":"d96c96e4ecf5fcfe45d2838d27a9d918","index":{"nums":[0,1,140,0,70,18,0,14],"html_filename":"d_af97b5493da09a14_patching_py.html","relative_filename":"transformer_lens/patching.py"}},"d_712808f24eb400fe___init___py":{"hash":"ca2411bb2d5283cba14da3ce60f5feec","index":{"nums":[0,1,0,0,0,0,0,0],"html_filename":"d_712808f24eb400fe___init___py.html","relative_filename":"transformer_lens/pretrained/__init__.py"}},"d_c1ea89878f9b2ac7___init___py":{"hash":"2994f6b388ff62b0d84269971b0766e3","index":{"nums":[0,1,20,0,0,0,0,0],"html_filename":"d_c1ea89878f9b2ac7___init___py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/__init__.py"}},"d_c1ea89878f9b2ac7_bert_py":{"hash":"1de2f9e5bcd80d6bcc6794f7073bbe66","index":{"nums":[0,1,31,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_bert_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/bert.py"}},"d_c1ea89878f9b2ac7_bloom_py":{"hash":"28eee61cd9581bbb556c20abd44717d1","index":{"nums":[0,1,40,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_bloom_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/bloom.py"}},"d_c1ea89878f9b2ac7_coder_py":{"hash":"008f8c07ead34001e59d834e9672b208","index":{"nums":[0,1,43,0,38,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_coder_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/coder.py"}},"d_c1ea89878f9b2ac7_gemma_py":{"hash":"b3ee2563edaf1aec4f1db3777cf34c84","index":{"nums":[0,1,41,0,36,6,0,6],"html_filename":"d_c1ea89878f9b2ac7_gemma_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/gemma.py"}},"d_c1ea89878f9b2ac7_gpt2_py":{"hash":"b6864d10cf3b70f76be00e47695dbec2","index":{"nums":[0,1,39,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_gpt2_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/gpt2.py"}},"d_c1ea89878f9b2ac7_gptj_py":{"hash":"6b6d95ba6133fe6935f33c2431cec376","index":{"nums":[0,1,36,0,31,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_gptj_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/gptj.py"}},"d_c1ea89878f9b2ac7_llama_py":{"hash":"b283de32d58e72f23d1dde8b2a54f278","index":{"nums":[0,1,45,0,39,8,0,8],"html_filename":"d_c1ea89878f9b2ac7_llama_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/llama.py"}},"d_c1ea89878f9b2ac7_mingpt_py":{"hash":"8afdb1ccd308a76b7af9ea71404ed112","index":{"nums":[0,1,40,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_mingpt_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/mingpt.py"}},"d_c1ea89878f9b2ac7_mistral_py":{"hash":"f08ee3cdaa7c2dfb152ef82a4917497f","index":{"nums":[0,1,36,0,31,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_mistral_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/mistral.py"}},"d_c1ea89878f9b2ac7_mixtral_py":{"hash":"bb26007d26637b02b3869d4733d0d6e8","index":{"nums":[0,1,37,0,32,4,0,4],"html_filename":"d_c1ea89878f9b2ac7_mixtral_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/mixtral.py"}},"d_c1ea89878f9b2ac7_nanogpt_py":{"hash":"e2ff3818963256534c9ca6a4ff6566a5","index":{"nums":[0,1,52,0,47,10,0,10],"html_filename":"d_c1ea89878f9b2ac7_nanogpt_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/nanogpt.py"}},"d_c1ea89878f9b2ac7_neel_solu_old_py":{"hash":"d96be68dd2fba5dbcba015e7f8b10fc1","index":{"nums":[0,1,17,0,0,12,2,2],"html_filename":"d_c1ea89878f9b2ac7_neel_solu_old_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/neel_solu_old.py"}},"d_c1ea89878f9b2ac7_neo_py":{"hash":"67f2d6bb1adda03d8d3eacbe5e1a0cb4","index":{"nums":[0,1,37,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_neo_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/neo.py"}},"d_c1ea89878f9b2ac7_neox_py":{"hash":"fde01ee4309dd58bf3a6793ca3421f9e","index":{"nums":[0,1,34,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_neox_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/neox.py"}},"d_c1ea89878f9b2ac7_opt_py":{"hash":"5bd938981ffdf863cddf1e54a3526615","index":{"nums":[0,1,40,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_opt_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/opt.py"}},"d_c1ea89878f9b2ac7_phi_py":{"hash":"487a556d6755479e53397cfa39646490","index":{"nums":[0,1,41,0,37,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_phi_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/phi.py"}},"d_c1ea89878f9b2ac7_phi3_py":{"hash":"9d833a74a1082fc28aa204b928511d91","index":{"nums":[0,1,35,0,30,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_phi3_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/phi3.py"}},"d_c1ea89878f9b2ac7_qwen_py":{"hash":"5abcfa29a1d0456fb304a4b3b7ee2063","index":{"nums":[0,1,38,0,33,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_qwen_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/qwen.py"}},"d_c1ea89878f9b2ac7_qwen2_py":{"hash":"5b12a2fd057e7fa3027aa59a9f629409","index":{"nums":[0,1,41,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_qwen2_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/qwen2.py"}},"d_c1ea89878f9b2ac7_t5_py":{"hash":"fb47f0cb69cfc6778072b400fe6e53ef","index":{"nums":[0,1,33,0,0,4,0,0],"html_filename":"d_c1ea89878f9b2ac7_t5_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/t5.py"}},"d_af97b5493da09a14_train_py":{"hash":"74c4dc0140e6c628f623287b84708fb2","index":{"nums":[0,1,72,0,42,34,1,33],"html_filename":"d_af97b5493da09a14_train_py.html","relative_filename":"transformer_lens/train.py"}},"d_b2114f845e0399b7___init___py":{"hash":"ca2411bb2d5283cba14da3ce60f5feec","index":{"nums":[0,1,0,0,0,0,0,0],"html_filename":"d_b2114f845e0399b7___init___py.html","relative_filename":"transformer_lens/utilities/__init__.py"}},"d_b2114f845e0399b7_activation_functions_py":{"hash":"0926a004863506fd686e3d1e0ae5eb84","index":{"nums":[0,1,6,0,0,2,0,0],"html_filename":"d_b2114f845e0399b7_activation_functions_py.html","relative_filename":"transformer_lens/utilities/activation_functions.py"}},"d_b2114f845e0399b7_addmm_py":{"hash":"b1cb8300653ca4c3dd6a4f5c06408a84","index":{"nums":[0,1,10,0,0,0,0,0],"html_filename":"d_b2114f845e0399b7_addmm_py.html","relative_filename":"transformer_lens/utilities/addmm.py"}},"d_b2114f845e0399b7_attention_py":{"hash":"a1d30d709e8539cc4515e76ad414a3ae","index":{"nums":[0,1,14,0,0,0,0,0],"html_filename":"d_b2114f845e0399b7_attention_py.html","relative_filename":"transformer_lens/utilities/attention.py"}},"d_b2114f845e0399b7_devices_py":{"hash":"1da1c1a01ae3a7697bbb04016881df11","index":{"nums":[0,1,31,0,7,18,4,10],"html_filename":"d_b2114f845e0399b7_devices_py.html","relative_filename":"transformer_lens/utilities/devices.py"}},"d_af97b5493da09a14_utils_py":{"hash":"d2b4d5cb90c365cbb3ef3981a81e5ae9","index":{"nums":[0,1,458,0,135,208,18,88],"html_filename":"d_af97b5493da09a14_utils_py.html","relative_filename":"transformer_lens/utils.py"}}}}
\ No newline at end of file
diff --git a/generated/code/transformer_lens.loading_from_pretrained.html b/generated/code/transformer_lens.loading_from_pretrained.html
index 3dd320b86..7ddfeca5f 100644
--- a/generated/code/transformer_lens.loading_from_pretrained.html
+++ b/generated/code/transformer_lens.loading_from_pretrained.html
@@ -370,7 +370,7 @@
 
 <dl class="py data">
 <dt class="sig sig-object py" id="transformer_lens.loading_from_pretrained.OFFICIAL_MODEL_NAMES">
-<span class="sig-prename descclassname"><span class="pre">transformer_lens.loading_from_pretrained.</span></span><span class="sig-name descname"><span class="pre">OFFICIAL_MODEL_NAMES</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">['gpt2',</span> <span class="pre">'gpt2-medium',</span> <span class="pre">'gpt2-large',</span> <span class="pre">'gpt2-xl',</span> <span class="pre">'distilgpt2',</span> <span class="pre">'facebook/opt-125m',</span> <span class="pre">'facebook/opt-1.3b',</span> <span class="pre">'facebook/opt-2.7b',</span> <span class="pre">'facebook/opt-6.7b',</span> <span class="pre">'facebook/opt-13b',</span> <span class="pre">'facebook/opt-30b',</span> <span class="pre">'facebook/opt-66b',</span> <span class="pre">'EleutherAI/gpt-neo-125M',</span> <span class="pre">'EleutherAI/gpt-neo-1.3B',</span> <span class="pre">'EleutherAI/gpt-neo-2.7B',</span> <span class="pre">'EleutherAI/gpt-j-6B',</span> <span class="pre">'EleutherAI/gpt-neox-20b',</span> <span class="pre">'stanford-crfm/alias-gpt2-small-x21',</span> <span class="pre">'stanford-crfm/battlestar-gpt2-small-x49',</span> <span class="pre">'stanford-crfm/caprica-gpt2-small-x81',</span> <span class="pre">'stanford-crfm/darkmatter-gpt2-small-x343',</span> <span class="pre">'stanford-crfm/expanse-gpt2-small-x777',</span> <span class="pre">'stanford-crfm/arwen-gpt2-medium-x21',</span> <span class="pre">'stanford-crfm/beren-gpt2-medium-x49',</span> <span class="pre">'stanford-crfm/celebrimbor-gpt2-medium-x81',</span> <span class="pre">'stanford-crfm/durin-gpt2-medium-x343',</span> <span class="pre">'stanford-crfm/eowyn-gpt2-medium-x777',</span> <span class="pre">'EleutherAI/pythia-14m',</span> <span class="pre">'EleutherAI/pythia-31m',</span> <span class="pre">'EleutherAI/pythia-70m',</span> <span class="pre">'EleutherAI/pythia-160m',</span> <span class="pre">'EleutherAI/pythia-410m',</span> <span class="pre">'EleutherAI/pythia-1b',</span> <span class="pre">'EleutherAI/pythia-1.4b',</span> <span class="pre">'EleutherAI/pythia-2.8b',</span> <span class="pre">'EleutherAI/pythia-6.9b',</span> <span class="pre">'EleutherAI/pythia-12b',</span> <span class="pre">'EleutherAI/pythia-70m-deduped',</span> <span class="pre">'EleutherAI/pythia-160m-deduped',</span> <span class="pre">'EleutherAI/pythia-410m-deduped',</span> <span class="pre">'EleutherAI/pythia-1b-deduped',</span> <span class="pre">'EleutherAI/pythia-1.4b-deduped',</span> <span class="pre">'EleutherAI/pythia-2.8b-deduped',</span> <span class="pre">'EleutherAI/pythia-6.9b-deduped',</span> <span class="pre">'EleutherAI/pythia-12b-deduped',</span> <span class="pre">'EleutherAI/pythia-70m-v0',</span> <span class="pre">'EleutherAI/pythia-160m-v0',</span> <span class="pre">'EleutherAI/pythia-410m-v0',</span> <span class="pre">'EleutherAI/pythia-1b-v0',</span> <span class="pre">'EleutherAI/pythia-1.4b-v0',</span> <span class="pre">'EleutherAI/pythia-2.8b-v0',</span> <span class="pre">'EleutherAI/pythia-6.9b-v0',</span> <span class="pre">'EleutherAI/pythia-12b-v0',</span> <span class="pre">'EleutherAI/pythia-70m-deduped-v0',</span> <span class="pre">'EleutherAI/pythia-160m-deduped-v0',</span> <span class="pre">'EleutherAI/pythia-410m-deduped-v0',</span> <span class="pre">'EleutherAI/pythia-1b-deduped-v0',</span> <span class="pre">'EleutherAI/pythia-1.4b-deduped-v0',</span> <span class="pre">'EleutherAI/pythia-2.8b-deduped-v0',</span> <span class="pre">'EleutherAI/pythia-6.9b-deduped-v0',</span> <span class="pre">'EleutherAI/pythia-12b-deduped-v0',</span> <span class="pre">'EleutherAI/pythia-160m-seed1',</span> <span class="pre">'EleutherAI/pythia-160m-seed2',</span> <span class="pre">'EleutherAI/pythia-160m-seed3',</span> <span class="pre">'NeelNanda/SoLU_1L_v9_old',</span> <span class="pre">'NeelNanda/SoLU_2L_v10_old',</span> <span class="pre">'NeelNanda/SoLU_4L_v11_old',</span> <span class="pre">'NeelNanda/SoLU_6L_v13_old',</span> <span class="pre">'NeelNanda/SoLU_8L_v21_old',</span> <span class="pre">'NeelNanda/SoLU_10L_v22_old',</span> <span class="pre">'NeelNanda/SoLU_12L_v23_old',</span> <span class="pre">'NeelNanda/SoLU_1L512W_C4_Code',</span> <span class="pre">'NeelNanda/SoLU_2L512W_C4_Code',</span> <span class="pre">'NeelNanda/SoLU_3L512W_C4_Code',</span> <span class="pre">'NeelNanda/SoLU_4L512W_C4_Code',</span> <span class="pre">'NeelNanda/SoLU_6L768W_C4_Code',</span> <span class="pre">'NeelNanda/SoLU_8L1024W_C4_Code',</span> <span class="pre">'NeelNanda/SoLU_10L1280W_C4_Code',</span> <span class="pre">'NeelNanda/SoLU_12L1536W_C4_Code',</span> <span class="pre">'NeelNanda/GELU_1L512W_C4_Code',</span> <span class="pre">'NeelNanda/GELU_2L512W_C4_Code',</span> <span class="pre">'NeelNanda/GELU_3L512W_C4_Code',</span> <span class="pre">'NeelNanda/GELU_4L512W_C4_Code',</span> <span class="pre">'NeelNanda/Attn_Only_1L512W_C4_Code',</span> <span class="pre">'NeelNanda/Attn_Only_2L512W_C4_Code',</span> <span class="pre">'NeelNanda/Attn_Only_3L512W_C4_Code',</span> <span class="pre">'NeelNanda/Attn_Only_4L512W_C4_Code',</span> <span class="pre">'NeelNanda/Attn-Only-2L512W-Shortformer-6B-big-lr',</span> <span class="pre">'NeelNanda/SoLU_1L512W_Wiki_Finetune',</span> <span class="pre">'NeelNanda/SoLU_4L512W_Wiki_Finetune',</span> <span class="pre">'ArthurConmy/redwood_attn_2l',</span> <span class="pre">'llama-7b-hf',</span> <span class="pre">'llama-13b-hf',</span> <span class="pre">'llama-30b-hf',</span> <span class="pre">'llama-65b-hf',</span> <span class="pre">'meta-llama/Llama-2-7b-hf',</span> <span class="pre">'meta-llama/Llama-2-7b-chat-hf',</span> <span class="pre">'meta-llama/Llama-2-13b-hf',</span> <span class="pre">'meta-llama/Llama-2-13b-chat-hf',</span> <span class="pre">'meta-llama/Llama-2-70b-chat-hf',</span> <span class="pre">'codellama/CodeLlama-7b-hf',</span> <span class="pre">'codellama/CodeLlama-7b-Python-hf',</span> <span class="pre">'codellama/CodeLlama-7b-Instruct-hf',</span> <span class="pre">'meta-llama/Meta-Llama-3-8B',</span> <span class="pre">'meta-llama/Meta-Llama-3-8B-Instruct',</span> <span class="pre">'meta-llama/Meta-Llama-3-70B',</span> <span class="pre">'meta-llama/Meta-Llama-3-70B-Instruct',</span> <span class="pre">'meta-llama/Llama-3.2-1B',</span> <span class="pre">'meta-llama/Llama-3.2-3B',</span> <span class="pre">'meta-llama/Llama-3.2-1B-Instruct',</span> <span class="pre">'meta-llama/Llama-3.2-3B-Instruct',</span> <span class="pre">'meta-llama/Llama-3.1-70B',</span> <span class="pre">'meta-llama/Llama-3.1-8B',</span> <span class="pre">'meta-llama/Llama-3.1-8B-Instruct',</span> <span class="pre">'meta-llama/Llama-3.1-70B-Instruct',</span> <span class="pre">'Baidicoot/Othello-GPT-Transformer-Lens',</span> <span class="pre">'bert-base-cased',</span> <span class="pre">'roneneldan/TinyStories-1M',</span> <span class="pre">'roneneldan/TinyStories-3M',</span> <span class="pre">'roneneldan/TinyStories-8M',</span> <span class="pre">'roneneldan/TinyStories-28M',</span> <span class="pre">'roneneldan/TinyStories-33M',</span> <span class="pre">'roneneldan/TinyStories-Instruct-1M',</span> <span class="pre">'roneneldan/TinyStories-Instruct-3M',</span> <span class="pre">'roneneldan/TinyStories-Instruct-8M',</span> <span class="pre">'roneneldan/TinyStories-Instruct-28M',</span> <span class="pre">'roneneldan/TinyStories-Instruct-33M',</span> <span class="pre">'roneneldan/TinyStories-1Layer-21M',</span> <span class="pre">'roneneldan/TinyStories-2Layers-33M',</span> <span class="pre">'roneneldan/TinyStories-Instuct-1Layer-21M',</span> <span class="pre">'roneneldan/TinyStories-Instruct-2Layers-33M',</span> <span class="pre">'stabilityai/stablelm-base-alpha-3b',</span> <span class="pre">'stabilityai/stablelm-base-alpha-7b',</span> <span class="pre">'stabilityai/stablelm-tuned-alpha-3b',</span> <span class="pre">'stabilityai/stablelm-tuned-alpha-7b',</span> <span class="pre">'mistralai/Mistral-7B-v0.1',</span> <span class="pre">'mistralai/Mistral-7B-Instruct-v0.1',</span> <span class="pre">'mistralai/Mistral-Nemo-Base-2407',</span> <span class="pre">'mistralai/Mixtral-8x7B-v0.1',</span> <span class="pre">'mistralai/Mixtral-8x7B-Instruct-v0.1',</span> <span class="pre">'bigscience/bloom-560m',</span> <span class="pre">'bigscience/bloom-1b1',</span> <span class="pre">'bigscience/bloom-1b7',</span> <span class="pre">'bigscience/bloom-3b',</span> <span class="pre">'bigscience/bloom-7b1',</span> <span class="pre">'bigcode/santacoder',</span> <span class="pre">'Qwen/Qwen-1_8B',</span> <span class="pre">'Qwen/Qwen-7B',</span> <span class="pre">'Qwen/Qwen-14B',</span> <span class="pre">'Qwen/Qwen-1_8B-Chat',</span> <span class="pre">'Qwen/Qwen-7B-Chat',</span> <span class="pre">'Qwen/Qwen-14B-Chat',</span> <span class="pre">'Qwen/Qwen1.5-0.5B',</span> <span class="pre">'Qwen/Qwen1.5-0.5B-Chat',</span> <span class="pre">'Qwen/Qwen1.5-1.8B',</span> <span class="pre">'Qwen/Qwen1.5-1.8B-Chat',</span> <span class="pre">'Qwen/Qwen1.5-4B',</span> <span class="pre">'Qwen/Qwen1.5-4B-Chat',</span> <span class="pre">'Qwen/Qwen1.5-7B',</span> <span class="pre">'Qwen/Qwen1.5-7B-Chat',</span> <span class="pre">'Qwen/Qwen1.5-14B',</span> <span class="pre">'Qwen/Qwen1.5-14B-Chat',</span> <span class="pre">'Qwen/Qwen2-0.5B',</span> <span class="pre">'Qwen/Qwen2-0.5B-Instruct',</span> <span class="pre">'Qwen/Qwen2-1.5B',</span> <span class="pre">'Qwen/Qwen2-1.5B-Instruct',</span> <span class="pre">'Qwen/Qwen2-7B',</span> <span class="pre">'Qwen/Qwen2-7B-Instruct',</span> <span class="pre">'Qwen/Qwen2.5-0.5B',</span> <span class="pre">'Qwen/Qwen2.5-0.5B-Instruct',</span> <span class="pre">'Qwen/Qwen2.5-1.5B',</span> <span class="pre">'Qwen/Qwen2.5-1.5B-Instruct',</span> <span class="pre">'Qwen/Qwen2.5-3B',</span> <span class="pre">'Qwen/Qwen2.5-3B-Instruct',</span> <span class="pre">'Qwen/Qwen2.5-7B',</span> <span class="pre">'Qwen/Qwen2.5-7B-Instruct',</span> <span class="pre">'Qwen/Qwen2.5-14B',</span> <span class="pre">'Qwen/Qwen2.5-14B-Instruct',</span> <span class="pre">'Qwen/Qwen2.5-32B',</span> <span class="pre">'Qwen/Qwen2.5-32B-Instruct',</span> <span class="pre">'Qwen/Qwen2.5-72B',</span> <span class="pre">'Qwen/Qwen2.5-72B-Instruct',</span> <span class="pre">'Qwen/QwQ-32B-Preview',</span> <span class="pre">'microsoft/phi-1',</span> <span class="pre">'microsoft/phi-1_5',</span> <span class="pre">'microsoft/phi-2',</span> <span class="pre">'microsoft/Phi-3-mini-4k-instruct',</span> <span class="pre">'google/gemma-2b',</span> <span class="pre">'google/gemma-7b',</span> <span class="pre">'google/gemma-2b-it',</span> <span class="pre">'google/gemma-7b-it',</span> <span class="pre">'google/gemma-2-2b',</span> <span class="pre">'google/gemma-2-2b-it',</span> <span class="pre">'google/gemma-2-9b',</span> <span class="pre">'google/gemma-2-9b-it',</span> <span class="pre">'google/gemma-2-27b',</span> <span class="pre">'google/gemma-2-27b-it',</span> <span class="pre">'01-ai/Yi-6B',</span> <span class="pre">'01-ai/Yi-34B',</span> <span class="pre">'01-ai/Yi-6B-Chat',</span> <span class="pre">'01-ai/Yi-34B-Chat',</span> <span class="pre">'google-t5/t5-small',</span> <span class="pre">'google-t5/t5-base',</span> <span class="pre">'google-t5/t5-large',</span> <span class="pre">'ai-forever/mGPT']</span></em><a class="headerlink" href="#transformer_lens.loading_from_pretrained.OFFICIAL_MODEL_NAMES" title="Permalink to this definition">#</a></dt>
+<span class="sig-prename descclassname"><span class="pre">transformer_lens.loading_from_pretrained.</span></span><span class="sig-name descname"><span class="pre">OFFICIAL_MODEL_NAMES</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">['gpt2',</span> <span class="pre">'gpt2-medium',</span> <span class="pre">'gpt2-large',</span> <span class="pre">'gpt2-xl',</span> <span class="pre">'distilgpt2',</span> <span class="pre">'facebook/opt-125m',</span> <span class="pre">'facebook/opt-1.3b',</span> <span class="pre">'facebook/opt-2.7b',</span> <span class="pre">'facebook/opt-6.7b',</span> <span class="pre">'facebook/opt-13b',</span> <span class="pre">'facebook/opt-30b',</span> <span class="pre">'facebook/opt-66b',</span> <span class="pre">'EleutherAI/gpt-neo-125M',</span> <span class="pre">'EleutherAI/gpt-neo-1.3B',</span> <span class="pre">'EleutherAI/gpt-neo-2.7B',</span> <span class="pre">'EleutherAI/gpt-j-6B',</span> <span class="pre">'EleutherAI/gpt-neox-20b',</span> <span class="pre">'stanford-crfm/alias-gpt2-small-x21',</span> <span class="pre">'stanford-crfm/battlestar-gpt2-small-x49',</span> <span class="pre">'stanford-crfm/caprica-gpt2-small-x81',</span> <span class="pre">'stanford-crfm/darkmatter-gpt2-small-x343',</span> <span class="pre">'stanford-crfm/expanse-gpt2-small-x777',</span> <span class="pre">'stanford-crfm/arwen-gpt2-medium-x21',</span> <span class="pre">'stanford-crfm/beren-gpt2-medium-x49',</span> <span class="pre">'stanford-crfm/celebrimbor-gpt2-medium-x81',</span> <span class="pre">'stanford-crfm/durin-gpt2-medium-x343',</span> <span class="pre">'stanford-crfm/eowyn-gpt2-medium-x777',</span> <span class="pre">'EleutherAI/pythia-14m',</span> <span class="pre">'EleutherAI/pythia-31m',</span> <span class="pre">'EleutherAI/pythia-70m',</span> <span class="pre">'EleutherAI/pythia-160m',</span> <span class="pre">'EleutherAI/pythia-410m',</span> <span class="pre">'EleutherAI/pythia-1b',</span> <span class="pre">'EleutherAI/pythia-1.4b',</span> <span class="pre">'EleutherAI/pythia-2.8b',</span> <span class="pre">'EleutherAI/pythia-6.9b',</span> <span class="pre">'EleutherAI/pythia-12b',</span> <span class="pre">'EleutherAI/pythia-70m-deduped',</span> <span class="pre">'EleutherAI/pythia-160m-deduped',</span> <span class="pre">'EleutherAI/pythia-410m-deduped',</span> <span class="pre">'EleutherAI/pythia-1b-deduped',</span> <span class="pre">'EleutherAI/pythia-1.4b-deduped',</span> <span class="pre">'EleutherAI/pythia-2.8b-deduped',</span> <span class="pre">'EleutherAI/pythia-6.9b-deduped',</span> <span class="pre">'EleutherAI/pythia-12b-deduped',</span> <span class="pre">'EleutherAI/pythia-70m-v0',</span> <span class="pre">'EleutherAI/pythia-160m-v0',</span> <span class="pre">'EleutherAI/pythia-410m-v0',</span> <span class="pre">'EleutherAI/pythia-1b-v0',</span> <span class="pre">'EleutherAI/pythia-1.4b-v0',</span> <span class="pre">'EleutherAI/pythia-2.8b-v0',</span> <span class="pre">'EleutherAI/pythia-6.9b-v0',</span> <span class="pre">'EleutherAI/pythia-12b-v0',</span> <span class="pre">'EleutherAI/pythia-70m-deduped-v0',</span> <span class="pre">'EleutherAI/pythia-160m-deduped-v0',</span> <span class="pre">'EleutherAI/pythia-410m-deduped-v0',</span> <span class="pre">'EleutherAI/pythia-1b-deduped-v0',</span> <span class="pre">'EleutherAI/pythia-1.4b-deduped-v0',</span> <span class="pre">'EleutherAI/pythia-2.8b-deduped-v0',</span> <span class="pre">'EleutherAI/pythia-6.9b-deduped-v0',</span> <span class="pre">'EleutherAI/pythia-12b-deduped-v0',</span> <span class="pre">'EleutherAI/pythia-160m-seed1',</span> <span class="pre">'EleutherAI/pythia-160m-seed2',</span> <span class="pre">'EleutherAI/pythia-160m-seed3',</span> <span class="pre">'NeelNanda/SoLU_1L_v9_old',</span> <span class="pre">'NeelNanda/SoLU_2L_v10_old',</span> <span class="pre">'NeelNanda/SoLU_4L_v11_old',</span> <span class="pre">'NeelNanda/SoLU_6L_v13_old',</span> <span class="pre">'NeelNanda/SoLU_8L_v21_old',</span> <span class="pre">'NeelNanda/SoLU_10L_v22_old',</span> <span class="pre">'NeelNanda/SoLU_12L_v23_old',</span> <span class="pre">'NeelNanda/SoLU_1L512W_C4_Code',</span> <span class="pre">'NeelNanda/SoLU_2L512W_C4_Code',</span> <span class="pre">'NeelNanda/SoLU_3L512W_C4_Code',</span> <span class="pre">'NeelNanda/SoLU_4L512W_C4_Code',</span> <span class="pre">'NeelNanda/SoLU_6L768W_C4_Code',</span> <span class="pre">'NeelNanda/SoLU_8L1024W_C4_Code',</span> <span class="pre">'NeelNanda/SoLU_10L1280W_C4_Code',</span> <span class="pre">'NeelNanda/SoLU_12L1536W_C4_Code',</span> <span class="pre">'NeelNanda/GELU_1L512W_C4_Code',</span> <span class="pre">'NeelNanda/GELU_2L512W_C4_Code',</span> <span class="pre">'NeelNanda/GELU_3L512W_C4_Code',</span> <span class="pre">'NeelNanda/GELU_4L512W_C4_Code',</span> <span class="pre">'NeelNanda/Attn_Only_1L512W_C4_Code',</span> <span class="pre">'NeelNanda/Attn_Only_2L512W_C4_Code',</span> <span class="pre">'NeelNanda/Attn_Only_3L512W_C4_Code',</span> <span class="pre">'NeelNanda/Attn_Only_4L512W_C4_Code',</span> <span class="pre">'NeelNanda/Attn-Only-2L512W-Shortformer-6B-big-lr',</span> <span class="pre">'NeelNanda/SoLU_1L512W_Wiki_Finetune',</span> <span class="pre">'NeelNanda/SoLU_4L512W_Wiki_Finetune',</span> <span class="pre">'ArthurConmy/redwood_attn_2l',</span> <span class="pre">'llama-7b-hf',</span> <span class="pre">'llama-13b-hf',</span> <span class="pre">'llama-30b-hf',</span> <span class="pre">'llama-65b-hf',</span> <span class="pre">'meta-llama/Llama-2-7b-hf',</span> <span class="pre">'meta-llama/Llama-2-7b-chat-hf',</span> <span class="pre">'meta-llama/Llama-2-13b-hf',</span> <span class="pre">'meta-llama/Llama-2-13b-chat-hf',</span> <span class="pre">'meta-llama/Llama-2-70b-chat-hf',</span> <span class="pre">'codellama/CodeLlama-7b-hf',</span> <span class="pre">'codellama/CodeLlama-7b-Python-hf',</span> <span class="pre">'codellama/CodeLlama-7b-Instruct-hf',</span> <span class="pre">'meta-llama/Meta-Llama-3-8B',</span> <span class="pre">'meta-llama/Meta-Llama-3-8B-Instruct',</span> <span class="pre">'meta-llama/Meta-Llama-3-70B',</span> <span class="pre">'meta-llama/Meta-Llama-3-70B-Instruct',</span> <span class="pre">'meta-llama/Llama-3.1-70B',</span> <span class="pre">'meta-llama/Llama-3.1-8B',</span> <span class="pre">'meta-llama/Llama-3.1-8B-Instruct',</span> <span class="pre">'meta-llama/Llama-3.1-70B-Instruct',</span> <span class="pre">'meta-llama/Llama-3.2-1B',</span> <span class="pre">'meta-llama/Llama-3.2-3B',</span> <span class="pre">'meta-llama/Llama-3.2-1B-Instruct',</span> <span class="pre">'meta-llama/Llama-3.2-3B-Instruct',</span> <span class="pre">'meta-llama/Llama-3.3-70B-Instruct',</span> <span class="pre">'Baidicoot/Othello-GPT-Transformer-Lens',</span> <span class="pre">'bert-base-cased',</span> <span class="pre">'roneneldan/TinyStories-1M',</span> <span class="pre">'roneneldan/TinyStories-3M',</span> <span class="pre">'roneneldan/TinyStories-8M',</span> <span class="pre">'roneneldan/TinyStories-28M',</span> <span class="pre">'roneneldan/TinyStories-33M',</span> <span class="pre">'roneneldan/TinyStories-Instruct-1M',</span> <span class="pre">'roneneldan/TinyStories-Instruct-3M',</span> <span class="pre">'roneneldan/TinyStories-Instruct-8M',</span> <span class="pre">'roneneldan/TinyStories-Instruct-28M',</span> <span class="pre">'roneneldan/TinyStories-Instruct-33M',</span> <span class="pre">'roneneldan/TinyStories-1Layer-21M',</span> <span class="pre">'roneneldan/TinyStories-2Layers-33M',</span> <span class="pre">'roneneldan/TinyStories-Instuct-1Layer-21M',</span> <span class="pre">'roneneldan/TinyStories-Instruct-2Layers-33M',</span> <span class="pre">'stabilityai/stablelm-base-alpha-3b',</span> <span class="pre">'stabilityai/stablelm-base-alpha-7b',</span> <span class="pre">'stabilityai/stablelm-tuned-alpha-3b',</span> <span class="pre">'stabilityai/stablelm-tuned-alpha-7b',</span> <span class="pre">'mistralai/Mistral-7B-v0.1',</span> <span class="pre">'mistralai/Mistral-7B-Instruct-v0.1',</span> <span class="pre">'mistralai/Mistral-Nemo-Base-2407',</span> <span class="pre">'mistralai/Mixtral-8x7B-v0.1',</span> <span class="pre">'mistralai/Mixtral-8x7B-Instruct-v0.1',</span> <span class="pre">'bigscience/bloom-560m',</span> <span class="pre">'bigscience/bloom-1b1',</span> <span class="pre">'bigscience/bloom-1b7',</span> <span class="pre">'bigscience/bloom-3b',</span> <span class="pre">'bigscience/bloom-7b1',</span> <span class="pre">'bigcode/santacoder',</span> <span class="pre">'Qwen/Qwen-1_8B',</span> <span class="pre">'Qwen/Qwen-7B',</span> <span class="pre">'Qwen/Qwen-14B',</span> <span class="pre">'Qwen/Qwen-1_8B-Chat',</span> <span class="pre">'Qwen/Qwen-7B-Chat',</span> <span class="pre">'Qwen/Qwen-14B-Chat',</span> <span class="pre">'Qwen/Qwen1.5-0.5B',</span> <span class="pre">'Qwen/Qwen1.5-0.5B-Chat',</span> <span class="pre">'Qwen/Qwen1.5-1.8B',</span> <span class="pre">'Qwen/Qwen1.5-1.8B-Chat',</span> <span class="pre">'Qwen/Qwen1.5-4B',</span> <span class="pre">'Qwen/Qwen1.5-4B-Chat',</span> <span class="pre">'Qwen/Qwen1.5-7B',</span> <span class="pre">'Qwen/Qwen1.5-7B-Chat',</span> <span class="pre">'Qwen/Qwen1.5-14B',</span> <span class="pre">'Qwen/Qwen1.5-14B-Chat',</span> <span class="pre">'Qwen/Qwen2-0.5B',</span> <span class="pre">'Qwen/Qwen2-0.5B-Instruct',</span> <span class="pre">'Qwen/Qwen2-1.5B',</span> <span class="pre">'Qwen/Qwen2-1.5B-Instruct',</span> <span class="pre">'Qwen/Qwen2-7B',</span> <span class="pre">'Qwen/Qwen2-7B-Instruct',</span> <span class="pre">'Qwen/Qwen2.5-0.5B',</span> <span class="pre">'Qwen/Qwen2.5-0.5B-Instruct',</span> <span class="pre">'Qwen/Qwen2.5-1.5B',</span> <span class="pre">'Qwen/Qwen2.5-1.5B-Instruct',</span> <span class="pre">'Qwen/Qwen2.5-3B',</span> <span class="pre">'Qwen/Qwen2.5-3B-Instruct',</span> <span class="pre">'Qwen/Qwen2.5-7B',</span> <span class="pre">'Qwen/Qwen2.5-7B-Instruct',</span> <span class="pre">'Qwen/Qwen2.5-14B',</span> <span class="pre">'Qwen/Qwen2.5-14B-Instruct',</span> <span class="pre">'Qwen/Qwen2.5-32B',</span> <span class="pre">'Qwen/Qwen2.5-32B-Instruct',</span> <span class="pre">'Qwen/Qwen2.5-72B',</span> <span class="pre">'Qwen/Qwen2.5-72B-Instruct',</span> <span class="pre">'Qwen/QwQ-32B-Preview',</span> <span class="pre">'microsoft/phi-1',</span> <span class="pre">'microsoft/phi-1_5',</span> <span class="pre">'microsoft/phi-2',</span> <span class="pre">'microsoft/Phi-3-mini-4k-instruct',</span> <span class="pre">'google/gemma-2b',</span> <span class="pre">'google/gemma-7b',</span> <span class="pre">'google/gemma-2b-it',</span> <span class="pre">'google/gemma-7b-it',</span> <span class="pre">'google/gemma-2-2b',</span> <span class="pre">'google/gemma-2-2b-it',</span> <span class="pre">'google/gemma-2-9b',</span> <span class="pre">'google/gemma-2-9b-it',</span> <span class="pre">'google/gemma-2-27b',</span> <span class="pre">'google/gemma-2-27b-it',</span> <span class="pre">'01-ai/Yi-6B',</span> <span class="pre">'01-ai/Yi-34B',</span> <span class="pre">'01-ai/Yi-6B-Chat',</span> <span class="pre">'01-ai/Yi-34B-Chat',</span> <span class="pre">'google-t5/t5-small',</span> <span class="pre">'google-t5/t5-base',</span> <span class="pre">'google-t5/t5-large',</span> <span class="pre">'ai-forever/mGPT']</span></em><a class="headerlink" href="#transformer_lens.loading_from_pretrained.OFFICIAL_MODEL_NAMES" title="Permalink to this definition">#</a></dt>
 <dd><p>Official model names for models on HuggingFace.</p>
 </dd></dl>
 
diff --git a/generated/demos/Exploratory_Analysis_Demo.html b/generated/demos/Exploratory_Analysis_Demo.html
index 596aa96c4..ee7229d67 100644
--- a/generated/demos/Exploratory_Analysis_Demo.html
+++ b/generated/demos/Exploratory_Analysis_Demo.html
@@ -476,37 +476,37 @@ <h3>Indirect Object Identification<a class="headerlink" href="#Indirect-Object-I
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "2566b58370ab4ee1a23ec40b298e524d"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "e3f6e035556c4a02a0f9ccad87c2f138"}</script></div>
 </div>
 <div class="nboutput docutils container">
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "4c4a1ec2b7514c1297506faae67c1374"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "7fd3dba640c2447299e8235086a74547"}</script></div>
 </div>
 <div class="nboutput docutils container">
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "54e358754fd241a7beb89dbd18d54469"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "7473a57e0b514125b34860321843604d"}</script></div>
 </div>
 <div class="nboutput docutils container">
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "c9ac7e38eb534678896df289998d1855"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "5af480c7ada14eb98937ccc79bb5be16"}</script></div>
 </div>
 <div class="nboutput docutils container">
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "0b727299d6b94848ba14d79205e0d847"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "fb98f7a2082142148838a198ceed048b"}</script></div>
 </div>
 <div class="nboutput docutils container">
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "cf0868984e46408fb4a1d566ec3db0b0"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "12f2342a4d7e48c9aaec6b2665da59ed"}</script></div>
 </div>
 <div class="nboutput nblast docutils container">
 <div class="prompt empty docutils container">
@@ -899,9 +899,9 @@ <h3>Logit Lens<a class="headerlink" href="#Logit-Lens" title="Permalink to this
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="49047634-c443-4608-b07a-015a3c6cf28b" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("49047634-c443-4608-b07a-015a3c6cf28b")) {                    Plotly.newPlot(                        "49047634-c443-4608-b07a-015a3c6cf28b",                        [{"hovertemplate":"\u003cb\u003e%{hovertext}\u003c\u002fb\u003e\u003cbr\u003e\u003cbr\u003ex=%{x}\u003cbr\u003ey=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","hovertext":["0_pre","0_mid","1_pre","1_mid","2_pre","2_mid","3_pre","3_mid","4_pre","4_mid","5_pre","5_mid","6_pre","6_mid","7_pre","7_mid","8_pre","8_mid","9_pre","9_mid","10_pre","10_mid","11_pre","11_mid","final_post"],"legendgroup":"","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines","name":"","orientation":"v","showlegend":false,"x":[0.0,0.5,1.0,1.5,2.0,2.5,3.0,3.5,4.0,4.5,5.0,5.5,6.0,6.5,7.0,7.5,8.0,8.5,9.0,9.5,10.0,10.5,11.0,11.5,12.0],"xaxis":"x","y":[1.2936594430357218e-05,-0.0066434419713914394,-0.007525080814957619,-0.009075661189854145,-0.008736724965274334,-0.008685516193509102,-0.006480449344962835,-0.007939981296658516,-0.009662019088864326,-0.01509616058319807,-0.014190619811415672,-0.019929546862840652,-0.009124485775828362,-0.02729814499616623,-0.02985497936606407,0.24972598254680634,0.25055843591690063,0.4500514566898346,0.4599689245223999,5.025448322296143,5.142899036407471,4.73056697845459,4.887067794799805,3.4453935623168945,3.5518839359283447],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"x"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"y"}},"legend":{"tracegroupgap":0},"title":{"text":"Logit Difference From Accumulate Residual Stream"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="515a6d53-e13c-4897-a00d-73d0be640d98" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("515a6d53-e13c-4897-a00d-73d0be640d98")) {                    Plotly.newPlot(                        "515a6d53-e13c-4897-a00d-73d0be640d98",                        [{"hovertemplate":"\u003cb\u003e%{hovertext}\u003c\u002fb\u003e\u003cbr\u003e\u003cbr\u003ex=%{x}\u003cbr\u003ey=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","hovertext":["0_pre","0_mid","1_pre","1_mid","2_pre","2_mid","3_pre","3_mid","4_pre","4_mid","5_pre","5_mid","6_pre","6_mid","7_pre","7_mid","8_pre","8_mid","9_pre","9_mid","10_pre","10_mid","11_pre","11_mid","final_post"],"legendgroup":"","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines","name":"","orientation":"v","showlegend":false,"x":[0.0,0.5,1.0,1.5,2.0,2.5,3.0,3.5,4.0,4.5,5.0,5.5,6.0,6.5,7.0,7.5,8.0,8.5,9.0,9.5,10.0,10.5,11.0,11.5,12.0],"xaxis":"x","y":[1.2936594430357218e-05,-0.0066434419713914394,-0.007525080814957619,-0.009075661189854145,-0.008736724965274334,-0.008685516193509102,-0.006480449344962835,-0.007939981296658516,-0.009662019088864326,-0.01509616058319807,-0.014190619811415672,-0.019929546862840652,-0.009124485775828362,-0.02729814499616623,-0.02985497936606407,0.24972598254680634,0.25055843591690063,0.4500514566898346,0.4599689245223999,5.025448322296143,5.142899036407471,4.73056697845459,4.887067794799805,3.4453935623168945,3.5518839359283447],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"x"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"y"}},"legend":{"tracegroupgap":0},"title":{"text":"Logit Difference From Accumulate Residual Stream"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('49047634-c443-4608-b07a-015a3c6cf28b');
+var gd = document.getElementById('515a6d53-e13c-4897-a00d-73d0be640d98');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -947,9 +947,9 @@ <h3>Layer Attribution<a class="headerlink" href="#Layer-Attribution" title="Perm
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="619fc2fb-6643-474d-9104-a4fc84996d47" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("619fc2fb-6643-474d-9104-a4fc84996d47")) {                    Plotly.newPlot(                        "619fc2fb-6643-474d-9104-a4fc84996d47",                        [{"hovertemplate":"\u003cb\u003e%{hovertext}\u003c\u002fb\u003e\u003cbr\u003e\u003cbr\u003ex=%{x}\u003cbr\u003ey=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","hovertext":["embed","pos_embed","0_attn_out","0_mlp_out","1_attn_out","1_mlp_out","2_attn_out","2_mlp_out","3_attn_out","3_mlp_out","4_attn_out","4_mlp_out","5_attn_out","5_mlp_out","6_attn_out","6_mlp_out","7_attn_out","7_mlp_out","8_attn_out","8_mlp_out","9_attn_out","9_mlp_out","10_attn_out","10_mlp_out","11_attn_out","11_mlp_out"],"legendgroup":"","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines","name":"","orientation":"v","showlegend":false,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25],"xaxis":"x","y":[-0.00028366505284793675,0.00029660051222890615,-0.006656361743807793,-0.0008817464113235474,-0.0015505198389291763,0.0003389097983017564,5.121063441038132e-05,0.002205097349360585,-0.001459577470086515,-0.0017220661975443363,-0.0054340846836566925,0.0009054935071617365,-0.005738964769989252,0.010805057361721992,-0.018173594027757645,-0.002556750550866127,0.27958089113235474,0.0008324328809976578,0.19949296116828918,0.009917395189404488,4.565478801727295,0.11744903028011322,-0.41232579946517944,0.1564953327178955,-1.4416704177856445,0.10648898035287857],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"x"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"y"}},"legend":{"tracegroupgap":0},"title":{"text":"Logit Difference From Each Layer"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="5929b143-3e55-4982-9cfd-dd66e0b08709" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("5929b143-3e55-4982-9cfd-dd66e0b08709")) {                    Plotly.newPlot(                        "5929b143-3e55-4982-9cfd-dd66e0b08709",                        [{"hovertemplate":"\u003cb\u003e%{hovertext}\u003c\u002fb\u003e\u003cbr\u003e\u003cbr\u003ex=%{x}\u003cbr\u003ey=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","hovertext":["embed","pos_embed","0_attn_out","0_mlp_out","1_attn_out","1_mlp_out","2_attn_out","2_mlp_out","3_attn_out","3_mlp_out","4_attn_out","4_mlp_out","5_attn_out","5_mlp_out","6_attn_out","6_mlp_out","7_attn_out","7_mlp_out","8_attn_out","8_mlp_out","9_attn_out","9_mlp_out","10_attn_out","10_mlp_out","11_attn_out","11_mlp_out"],"legendgroup":"","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines","name":"","orientation":"v","showlegend":false,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25],"xaxis":"x","y":[-0.00028366505284793675,0.00029660051222890615,-0.006656361743807793,-0.0008817464113235474,-0.0015505198389291763,0.0003389097983017564,5.121063441038132e-05,0.002205097349360585,-0.001459577470086515,-0.0017220661975443363,-0.0054340846836566925,0.0009054935071617365,-0.005738964769989252,0.010805057361721992,-0.018173594027757645,-0.002556750550866127,0.27958089113235474,0.0008324328809976578,0.19949296116828918,0.009917395189404488,4.565478801727295,0.11744903028011322,-0.41232579946517944,0.1564953327178955,-1.4416704177856445,0.10648898035287857],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"x"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"y"}},"legend":{"tracegroupgap":0},"title":{"text":"Logit Difference From Each Layer"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('619fc2fb-6643-474d-9104-a4fc84996d47');
+var gd = document.getElementById('5929b143-3e55-4982-9cfd-dd66e0b08709');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -1017,9 +1017,9 @@ <h2>Head Attribution<a class="headerlink" href="#Head-Attribution" title="Permal
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="d13d12cd-e58c-4918-adf2-4cba0fcd4cfb" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("d13d12cd-e58c-4918-adf2-4cba0fcd4cfb")) {                    Plotly.newPlot(                        "d13d12cd-e58c-4918-adf2-4cba0fcd4cfb",                        [{"coloraxis":"coloraxis","name":"0","z":[[-0.002056329045444727,-0.0005102002760395408,0.0004685759777203202,0.00012511858949437737,-0.0006028746138326824,-0.00024295759794767946,-0.0023188991472125053,-0.0027583539485931396,0.000564592657610774,0.0009697366622276604,-0.0002504411095287651,4.7396752052009106e-06],[-0.0010071106953546405,0.0003947066143155098,-0.001548723317682743,0.0014034844934940338,-0.0012652697041630745,-0.0011358254123479128,-0.0028159399516880512,-0.0029645331669598818,0.0029190238565206528,0.0025743518490344286,0.00036237656604498625,0.0017548904288560152],[0.0005569161148741841,-0.0011266364017501473,-0.0017354193842038512,-0.0014514740323647857,-0.0002874041674658656,0.0017210585065186024,0.0026659027207642794,0.0031146786641329527,0.0005667298682965338,-0.003666497301310301,-0.0018847138853743672,7.028633262962103e-06],[-0.0007264401647262275,0.0001136981591116637,0.0014301439514383674,0.0007490518037229776,0.002018478699028492,0.0007436765008606017,-0.0004617785452865064,-0.0039057875983417034,0.0011407355777919292,-4.027353134006262e-05,-0.0013293151278048754,-0.0017636835109442472],[-0.0028281747363507748,0.0003364539588801563,-0.0014249025844037533,-0.003777292789891362,0.001599886454641819,0.00029889593133702874,-0.0008046383736655116,0.0020388178527355194,-0.001559373107738793,-0.0006437154370360076,0.001116809668019414,-0.0003500327584333718],[0.0011338141048327088,0.0011259093880653381,-0.0025163700338453054,-0.0014790259301662445,0.00038791849510744214,-6.407558976206928e-05,-0.0005096771637909114,-0.0008841846138238907,0.0006399258272722363,-0.0010097046615555882,-0.006759032607078552,0.0033667325042188168],[-0.015147397294640541,-0.0021350914612412453,0.0025931934360414743,-0.00042673421557992697,-0.005559004843235016,0.002665933221578598,0.0064108846709132195,-0.0038270088844001293,-0.0003842375008389354,-0.0016430213581770658,-0.001334429020062089,-9.183748625218868e-05],[-9.488424984738231e-05,-0.005788922775536776,-0.0006383699947036803,0.134933739900589,-0.0017687629442662,-0.018917974084615707,0.0038733729161322117,-0.0021450738422572613,-0.010327240452170372,0.1832587718963623,-0.0007747872150503099,-0.0010452070273458958],[-0.0038339472375810146,-0.0008046274306252599,-0.01267334446310997,0.008045826107263565,0.003604009747505188,-0.009398158639669418,-0.08272106945514679,0.0035550370812416077,-0.01840413361787796,0.001758674974553287,0.28961312770843506,0.022854033857584],[0.08595199882984161,-0.000693259877152741,0.06816964596509933,0.01311142835766077,-0.02109798789024353,0.05112443491816521,1.384489893913269,0.045837316662073135,-0.038303449749946594,2.9854443073272705,0.001966138370335102,-0.008030208759009838],[0.5608724355697632,0.17083144187927246,-0.033618729561567307,0.05821548402309418,-0.0024530175141990185,0.0018771894974634051,0.2882729470729828,-1.8986413478851318,-0.0015286938287317753,-0.035130057483911514,0.48021769523620605,-0.0009116912842728198],[0.016075868159532547,-0.03986111283302307,-0.3879111707210541,0.01112320739775896,-0.0054778107441961765,-0.002512941136956215,-0.08056112378835678,0.007518645375967026,0.04301108419895172,-0.040082402527332306,-0.9702335000038147,0.011862380430102348]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Each Head"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="8ef8b5c5-d0eb-4f44-b5f8-1221b65bceb0" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("8ef8b5c5-d0eb-4f44-b5f8-1221b65bceb0")) {                    Plotly.newPlot(                        "8ef8b5c5-d0eb-4f44-b5f8-1221b65bceb0",                        [{"coloraxis":"coloraxis","name":"0","z":[[-0.002056329045444727,-0.0005102002760395408,0.0004685759777203202,0.00012511858949437737,-0.0006028746138326824,-0.00024295759794767946,-0.0023188991472125053,-0.0027583539485931396,0.000564592657610774,0.0009697366622276604,-0.0002504411095287651,4.7396752052009106e-06],[-0.0010071106953546405,0.0003947066143155098,-0.001548723317682743,0.0014034844934940338,-0.0012652697041630745,-0.0011358254123479128,-0.0028159399516880512,-0.0029645331669598818,0.0029190238565206528,0.0025743518490344286,0.00036237656604498625,0.0017548904288560152],[0.0005569161148741841,-0.0011266364017501473,-0.0017354193842038512,-0.0014514740323647857,-0.0002874041674658656,0.0017210585065186024,0.0026659027207642794,0.0031146786641329527,0.0005667298682965338,-0.003666497301310301,-0.0018847138853743672,7.028633262962103e-06],[-0.0007264401647262275,0.0001136981591116637,0.0014301439514383674,0.0007490518037229776,0.002018478699028492,0.0007436765008606017,-0.0004617785452865064,-0.0039057875983417034,0.0011407355777919292,-4.027353134006262e-05,-0.0013293151278048754,-0.0017636835109442472],[-0.0028281747363507748,0.0003364539588801563,-0.0014249025844037533,-0.003777292789891362,0.001599886454641819,0.00029889593133702874,-0.0008046383736655116,0.0020388178527355194,-0.001559373107738793,-0.0006437154370360076,0.001116809668019414,-0.0003500327584333718],[0.0011338141048327088,0.0011259093880653381,-0.0025163700338453054,-0.0014790259301662445,0.00038791849510744214,-6.407558976206928e-05,-0.0005096771637909114,-0.0008841846138238907,0.0006399258272722363,-0.0010097046615555882,-0.006759032607078552,0.0033667325042188168],[-0.015147397294640541,-0.0021350914612412453,0.0025931934360414743,-0.00042673421557992697,-0.005559004843235016,0.002665933221578598,0.0064108846709132195,-0.0038270088844001293,-0.0003842375008389354,-0.0016430213581770658,-0.001334429020062089,-9.183748625218868e-05],[-9.488424984738231e-05,-0.005788922775536776,-0.0006383699947036803,0.134933739900589,-0.0017687629442662,-0.018917974084615707,0.0038733729161322117,-0.0021450738422572613,-0.010327240452170372,0.1832587718963623,-0.0007747872150503099,-0.0010452070273458958],[-0.0038339472375810146,-0.0008046274306252599,-0.01267334446310997,0.008045826107263565,0.003604009747505188,-0.009398158639669418,-0.08272106945514679,0.0035550370812416077,-0.01840413361787796,0.001758674974553287,0.28961312770843506,0.022854033857584],[0.08595199882984161,-0.000693259877152741,0.06816964596509933,0.01311142835766077,-0.02109798789024353,0.05112443491816521,1.384489893913269,0.045837316662073135,-0.038303449749946594,2.9854443073272705,0.001966138370335102,-0.008030208759009838],[0.5608724355697632,0.17083144187927246,-0.033618729561567307,0.05821548402309418,-0.0024530175141990185,0.0018771894974634051,0.2882729470729828,-1.8986413478851318,-0.0015286938287317753,-0.035130057483911514,0.48021769523620605,-0.0009116912842728198],[0.016075868159532547,-0.03986111283302307,-0.3879111707210541,0.01112320739775896,-0.0054778107441961765,-0.002512941136956215,-0.08056112378835678,0.007518645375967026,0.04301108419895172,-0.040082402527332306,-0.9702335000038147,0.011862380430102348]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Each Head"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('d13d12cd-e58c-4918-adf2-4cba0fcd4cfb');
+var gd = document.getElementById('8ef8b5c5-d0eb-4f44-b5f8-1221b65bceb0');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -1141,19 +1141,19 @@ <h2>Attention Analysis<a class="headerlink" href="#Attention-Analysis" title="Pe
 </pre></div>
 </div>
 <div class="output_area rendered_html docutils container">
-<div style='max-width: 700px;'><h2>Top 3 Positive Logit Attribution Heads</h2><br/><div id="circuits-vis-238275f7-4d84" style="margin: 15px 0;"/>
+<div style='max-width: 700px;'><h2>Top 3 Positive Logit Attribution Heads</h2><br/><div id="circuits-vis-3a6547db-fa7c" style="margin: 15px 0;"/>
     <script crossorigin type="module">
     import { render, AttentionHeads } from "https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js";
     render(
-      "circuits-vis-238275f7-4d84",
+      "circuits-vis-3a6547db-fa7c",
       AttentionHeads,
       {"attention": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9979304075241089, 0.002069620881229639, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9971864819526672, 0.0010516609763726592, 0.0017618348356336355, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9594857096672058, 0.001310725579969585, 0.03694308176636696, 0.002260456094518304, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9891613125801086, 0.001052629784680903, 0.0048541901633143425, 0.001104357186704874, 0.003827564185485244, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9963598847389221, 0.0007981774397194386, 0.0007764195324853063, 0.00019262291607446969, 0.0002416159404674545, 0.0016313291853293777, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.981286346912384, 0.001863775309175253, 0.006344134453684092, 0.0002960922720376402, 0.004604941233992577, 0.0013966941041871905, 0.00420788861811161, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9950011372566223, 0.0013230532640591264, 0.0007920759962871671, 0.00024644460063427687, 0.0003405151073820889, 0.00016826140927150846, 0.0002837753272615373, 0.0018447580514475703, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9920291900634766, 0.0013539871433749795, 0.0007168339216150343, 9.214139572577551e-05, 0.00013419234892353415, 0.00019706363673321903, 0.00035278062568977475, 0.0002426079590804875, 0.004881155211478472, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5483196377754211, 0.0075838202610611916, 0.28265973925590515, 0.0029685653280466795, 0.14815661311149597, 0.0008172960951924324, 0.0006479909643530846, 0.0017078499076887965, 0.004129356704652309, 0.003009046893566847, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8642559051513672, 0.0036493062507361174, 0.004925842396914959, 0.06957440823316574, 0.038683053106069565, 0.009609837085008621, 0.0008548588375560939, 0.0005643004551529884, 0.003707138355821371, 0.0013952680164948106, 0.002780066104605794, 0.0, 0.0, 0.0, 0.0], [0.30252957344055176, 0.006584456656128168, 0.1401754468679428, 0.030065739527344704, 0.4806697964668274, 0.0005923578282818198, 0.0005309724947437644, 0.0024134721606969833, 0.008505754172801971, 0.0013766245683655143, 0.023764921352267265, 0.0027908117044717073, 0.0, 0.0, 0.0], [0.7773900032043457, 0.003722917754203081, 0.022378109395503998, 0.015139483846724033, 0.017408354207873344, 0.002416662173345685, 0.000709561922121793, 0.0007399603491649032, 0.13850137591362, 0.0023606533650308847, 0.005776867736130953, 0.0028014343697577715, 0.010654616169631481, 0.0, 0.0], [0.9735962152481079, 0.0012812522472813725, 0.002617918187752366, 9.891873924061656e-05, 0.0005098494002595544, 0.00012000725837424397, 0.00045229491661302745, 0.00010779645526781678, 0.002954537281766534, 0.001644388772547245, 0.0012141242623329163, 0.00018637391622178257, 0.0008510759216733277, 0.01436527632176876, 0.0], [0.0990300253033638, 0.000971083587501198, 0.06413238495588303, 0.005246965680271387, 0.8110400438308716, 8.250321843661368e-05, 6.596777529921383e-05, 0.0011991349747404456, 0.0008852760074660182, 0.00024891181965358555, 0.010336406528949738, 7.952339365147054e-05, 0.0049433172680437565, 0.00028918671887367964, 0.0014492359478026628]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9985041618347168, 0.0014959010295569897, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9966639876365662, 0.00046957843005657196, 0.002866449998691678, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9208283424377441, 0.0007845019572414458, 0.07676984369754791, 0.001617342815734446, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9835838675498962, 0.0006018307176418602, 0.0030122774187475443, 0.006188061088323593, 0.0066139851696789265, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9950478076934814, 0.0007798039587214589, 0.0007059435010887682, 0.0003824840241577476, 0.0007798775332048535, 0.0023041421081870794, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9838821887969971, 0.0015907409833744168, 0.0063361418433487415, 0.0004584550915751606, 0.004829846788197756, 0.0009966548532247543, 0.0019060741178691387, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.995481014251709, 0.0010436347220093012, 0.0005037460359744728, 0.0005756175960414112, 0.00046194688184186816, 0.0004267194017302245, 0.0006751787732355297, 0.0008320942288264632, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9798213243484497, 0.008518284186720848, 0.0010190301109105349, 0.00029932931647635996, 0.0003097986045759171, 0.0003330526524223387, 0.001541984616778791, 0.0008126517641358078, 0.007344536483287811, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6489241123199463, 0.008722440339624882, 0.10265976935625076, 0.0038905434776097536, 0.22694909572601318, 0.0011350901331752539, 0.00045881845289841294, 0.0004216691595502198, 0.0053738984279334545, 0.0014645822811871767, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9156036972999573, 0.0013435552828013897, 0.0011673923581838608, 0.04532041400671005, 0.020157871767878532, 0.007576430682092905, 0.0011305580846965313, 0.00024880640557967126, 0.004846310243010521, 0.0007249554619193077, 0.0018800381803885102, 0.0, 0.0, 0.0, 0.0], [0.4962114989757538, 0.0019782893359661102, 0.05844113230705261, 0.00783691368997097, 0.40375715494155884, 0.0005728096584789455, 0.0002963987353723496, 0.0009752871701493859, 0.0046507869847118855, 0.00032523463596589863, 0.02318551018834114, 0.0017689995002001524, 0.0, 0.0, 0.0], [0.8397698402404785, 0.0018636900931596756, 0.005577529780566692, 0.020380759611725807, 0.0292733795940876, 0.004448764491826296, 0.0011503315763548017, 0.00024191653938032687, 0.08715140074491501, 0.0013588638976216316, 0.0026416215114295483, 0.0019266613526269794, 0.004215260501950979, 0.0, 0.0], [0.9744170308113098, 0.0024021149147301912, 0.00455052312463522, 0.00020547708845697343, 0.002662503393366933, 0.00021827162709087133, 0.0008242498734034598, 0.0004054978198837489, 0.002643037121742964, 0.0015235628234222531, 0.0031088448595255613, 7.676492532482371e-05, 0.0018502527382224798, 0.0051117767579853535, 0.0], [0.18470405042171478, 0.00038850627606734633, 0.06518032401800156, 0.0014534658985212445, 0.7318763732910156, 0.00010888298129430041, 2.748575025179889e-05, 0.00011231198004679754, 0.0013672056375071406, 4.0162569348467514e-05, 0.012543701566755772, 5.95860110479407e-05, 0.0007756491540931165, 0.0010575354099273682, 0.00030486442847177386]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9776753783226013, 0.022324683144688606, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9430753588676453, 0.018823042511940002, 0.038101643323898315, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8963208794593811, 0.006171742454171181, 0.07676312327384949, 0.02074429951608181, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8522229790687561, 0.017844438552856445, 0.05478672310709953, 0.012976272031664848, 0.062169551849365234, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9669023752212524, 0.004049547016620636, 0.002062328392639756, 0.007766399998217821, 0.007081815041601658, 0.012137639336287975, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7676379680633545, 0.018502332270145416, 0.045013245195150375, 0.012471744790673256, 0.10591386258602142, 0.018323613330721855, 0.03213733434677124, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.891828179359436, 0.010069617070257664, 0.013090009801089764, 0.010685192421078682, 0.031169477850198746, 0.006502270698547363, 0.02103857509791851, 0.015616719610989094, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9457208514213562, 0.0045005157589912415, 0.013905955478549004, 0.0013248942559584975, 0.01557664293795824, 0.001115700462833047, 0.0007837467710487545, 0.0007244577864184976, 0.016347244381904602, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5131928324699402, 0.006032112520188093, 0.11406763643026352, 0.009519217535853386, 0.30613386631011963, 0.0034422571770846844, 0.002601674757897854, 0.002665397012606263, 0.03412351757287979, 0.008221502415835857, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8395131230354309, 0.008326224982738495, 0.012920893728733063, 0.01282776053994894, 0.029934609308838844, 0.013884111307561398, 0.0034839101135730743, 0.0018691613804548979, 0.039465710520744324, 0.011843967251479626, 0.025930611416697502, 0.0, 0.0, 0.0, 0.0], [0.3585011661052704, 0.004236206877976656, 0.10680253803730011, 0.015598620288074017, 0.41607996821403503, 0.002491925610229373, 0.003585703205317259, 0.0038999069947749376, 0.015854908153414726, 0.005938141606748104, 0.062027692794799805, 0.004983273800462484, 0.0, 0.0, 0.0], [0.612542450428009, 0.004679449368268251, 0.029155218973755836, 0.028240399435162544, 0.08258962631225586, 0.007909866981208324, 0.011129351332783699, 0.0034775957465171814, 0.17027747631072998, 0.010538897477090359, 0.015773506835103035, 0.013295088894665241, 0.01039104349911213, 0.0, 0.0], [0.875810444355011, 0.007089323364198208, 0.01208286453038454, 0.0025996132753789425, 0.021269435063004494, 0.00165357009973377, 0.0019036760786548257, 0.0016310046194121242, 0.02823040634393692, 0.008473552763462067, 0.011080354452133179, 0.002760963048785925, 0.0025940914638340473, 0.02282082475721836, 0.0], [0.5415229201316833, 0.003397508757188916, 0.03888256847858429, 0.007577123120427132, 0.3613935708999634, 0.001381068374030292, 0.0010475177550688386, 0.0012340175453573465, 0.006947703659534454, 0.003757019527256489, 0.0176718570291996, 0.0011743978830054402, 0.0036232525017112494, 0.00219246419146657, 0.008196980692446232]]], "attentionHeadNames": ["L9H9", "L9H6", "L10H0"], "tokens": ["<|endoftext|>", "When", " John", " and", " Mary", " went", " to", " the", " shops", ",", " John", " gave", " the", " bag", " to"]}
     )
-    </script></div><div style='max-width: 700px;'><h2>Top 3 Negative Logit Attribution Heads</h2><br/><div id="circuits-vis-a59ac87b-fe56" style="margin: 15px 0;"/>
+    </script></div><div style='max-width: 700px;'><h2>Top 3 Negative Logit Attribution Heads</h2><br/><div id="circuits-vis-f26fcf72-5687" style="margin: 15px 0;"/>
     <script crossorigin type="module">
     import { render, AttentionHeads } from "https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js";
     render(
-      "circuits-vis-a59ac87b-fe56",
+      "circuits-vis-f26fcf72-5687",
       AttentionHeads,
       {"attention": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9628916382789612, 0.03710832819342613, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9778335094451904, 0.0034864130429923534, 0.018680110573768616, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8737697601318359, 0.005964328069239855, 0.08126737922430038, 0.03899851813912392, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8745138645172119, 0.017276111990213394, 0.018874678760766983, 0.054268334060907364, 0.035066962242126465, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9704693555831909, 0.0035177830141037703, 0.0003239834331907332, 0.004373463336378336, 0.0003953034174628556, 0.02092011272907257, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7694658041000366, 0.006685878615826368, 0.011966485530138016, 0.040966276079416275, 0.03344443440437317, 0.04401002824306488, 0.0934610366821289, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8177722692489624, 0.009112595580518246, 0.0042336746118962765, 0.03900325670838356, 0.012653318233788013, 0.007337852846831083, 0.07410021126270294, 0.0357869453728199, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8309933543205261, 0.018796611577272415, 0.00330327614210546, 0.01983213610947132, 0.004009497817605734, 0.012078307569026947, 0.03246007487177849, 0.010322270914912224, 0.06820447742938995, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.31403300166130066, 0.01522789802402258, 0.19789263606071472, 0.059379592537879944, 0.3236042559146881, 0.009037697687745094, 0.019917670637369156, 0.012475269846618176, 0.013939663767814636, 0.03449229896068573, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7319568991661072, 0.03087025322020054, 0.005064091179519892, 0.08703826367855072, 0.004982746671885252, 0.044725559651851654, 0.02350614219903946, 0.0063977413810789585, 0.0034696452785283327, 0.048653144389390945, 0.013335512951016426, 0.0, 0.0, 0.0, 0.0], [0.13710583746433258, 0.012017901986837387, 0.14592327177524567, 0.015683427453041077, 0.44125232100486755, 0.004342416767030954, 0.011117888614535332, 0.007466601673513651, 0.0012341630645096302, 0.01946023479104042, 0.16842849552631378, 0.035967420786619186, 0.0, 0.0, 0.0], [0.4052484929561615, 0.009720955975353718, 0.03158125653862953, 0.09247038513422012, 0.048808369785547256, 0.008174203336238861, 0.031708307564258575, 0.02787904627621174, 0.14128026366233826, 0.0302340779453516, 0.03392859548330307, 0.03171432018280029, 0.10725171118974686, 0.0, 0.0], [0.808469295501709, 0.0056702690199017525, 0.006755992770195007, 0.008696584962308407, 0.010473440401256084, 0.003464736510068178, 0.006760997697710991, 0.0025889205280691385, 0.015097705647349358, 0.01603602059185505, 0.011288060806691647, 0.017277495935559273, 0.00779919745400548, 0.07962135970592499, 0.0], [0.02939111366868019, 0.0016060526249930263, 0.06005561351776123, 0.008175727911293507, 0.8099164366722107, 0.00039796155760996044, 0.0028282543644309044, 0.0033632616978138685, 0.0001569116284372285, 0.0036459483671933413, 0.05494074895977974, 0.001666039228439331, 0.015368940308690071, 5.016689101466909e-05, 0.008436810225248337]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9124024510383606, 0.08759760111570358, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.97407466173172, 0.01434413529932499, 0.011581187136471272, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8240247964859009, 0.03700513392686844, 0.06643965095281601, 0.07253041118383408, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5370795726776123, 0.15483324229717255, 0.0903128981590271, 0.12131187319755554, 0.0964624434709549, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9388002157211304, 0.028733234852552414, 0.0003431019140407443, 0.010481251403689384, 0.000839930260553956, 0.02080223336815834, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6318807005882263, 0.059302832931280136, 0.03517911583185196, 0.0317409411072731, 0.1174689382314682, 0.06869616359472275, 0.05573126673698425, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7625216841697693, 0.04435169696807861, 0.010552966967225075, 0.034276288002729416, 0.014203979633748531, 0.012703591957688332, 0.07259687036275864, 0.048792947083711624, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5725991725921631, 0.12681783735752106, 0.03727060556411743, 0.0431414395570755, 0.04129580408334732, 0.046447426080703735, 0.029164383187890053, 0.03751189261674881, 0.06575141847133636, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12316936254501343, 0.026347072795033455, 0.27960509061813354, 0.041989635676145554, 0.44105827808380127, 0.006138973869383335, 0.007793003693223, 0.007175281643867493, 0.011286810971796513, 0.055436521768569946, 0.0, 0.0, 0.0, 0.0, 0.0], [0.44855940341949463, 0.1432485729455948, 0.010774216614663601, 0.07855041325092316, 0.01404787041246891, 0.014864671975374222, 0.008633698336780071, 0.0030146560166031122, 0.012792127206921577, 0.16408587992191315, 0.10142850130796432, 0.0, 0.0, 0.0, 0.0], [0.10228201746940613, 0.0238832775503397, 0.055250268429517746, 0.0520680733025074, 0.5759841799736023, 0.0011948152678087354, 0.0013662497512996197, 0.0025677781086415052, 0.0018699930515140295, 0.043160032480955124, 0.10887196660041809, 0.03150142729282379, 0.0, 0.0, 0.0], [0.39313194155693054, 0.04406267777085304, 0.04009911045432091, 0.07407406717538834, 0.04061643034219742, 0.007346798665821552, 0.009507199749350548, 0.013865168206393719, 0.0640539675951004, 0.04894229769706726, 0.0627753809094429, 0.0999651551246643, 0.10155977308750153, 0.0, 0.0], [0.7573198676109314, 0.015095721930265427, 0.007350177504122257, 0.013156400062143803, 0.005726401228457689, 0.008556416258215904, 0.008282607421278954, 0.010307075455784798, 0.002904881490394473, 0.017329057678580284, 0.015188980847597122, 0.0796574279665947, 0.030244464054703712, 0.02888045459985733, 0.0], [0.08579524606466293, 0.023773644119501114, 0.028759371489286423, 0.050303805619478226, 0.674644947052002, 0.0010763936443254352, 0.0011514866491779685, 0.002721975091844797, 0.0005617713322862983, 0.039091579616069794, 0.04861969128251076, 0.015520088374614716, 0.0040845321491360664, 0.00043291584006510675, 0.023462524637579918]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9691730737686157, 0.03082684986293316, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9864587187767029, 0.0076598357409238815, 0.00588154187425971, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9318376183509827, 0.011915593408048153, 0.02053167298436165, 0.03571505844593048, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8786704540252686, 0.03427596017718315, 0.01315787062048912, 0.02705124393105507, 0.04684450104832649, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9850097298622131, 0.00501946872100234, 0.0013920071069151163, 0.0018760499078780413, 0.004171107430011034, 0.002531560370698571, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8907089829444885, 0.02026253007352352, 0.020904401317238808, 0.009134626016020775, 0.04057806357741356, 0.003215113654732704, 0.015196278691291809, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9318532347679138, 0.012423796579241753, 0.007052858360111713, 0.00626762630417943, 0.009715708903968334, 0.0013882736675441265, 0.012697475962340832, 0.018601125106215477, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7722940444946289, 0.03829518333077431, 0.03943629562854767, 0.00972924567759037, 0.11019386351108551, 0.009773456491529942, 0.00886810664087534, 0.003977742046117783, 0.007432003039866686, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6659272909164429, 0.019893741235136986, 0.05748209357261658, 0.026046503335237503, 0.1897869110107422, 0.001734035206027329, 0.004116130527108908, 0.004545124247670174, 0.007496052421629429, 0.022972112521529198, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8588331341743469, 0.01852157525718212, 0.010080324485898018, 0.0161746833473444, 0.044801417738199234, 0.018235526978969574, 0.0026932875625789165, 0.0009044647449627519, 0.004338268656283617, 0.01020037941634655, 0.015216901898384094, 0.0, 0.0, 0.0, 0.0], [0.7452806234359741, 0.02059219218790531, 0.033448558300733566, 0.04560310021042824, 0.09397682547569275, 0.0019108005799353123, 0.0039161862805485725, 0.004672198090702295, 0.003216084325686097, 0.0180679801851511, 0.0163005068898201, 0.013014941476285458, 0.0, 0.0, 0.0], [0.7450795769691467, 0.016002941876649857, 0.02564326673746109, 0.03000519424676895, 0.06691146641969681, 0.0032243167515844107, 0.006530732847750187, 0.006865624338388443, 0.04355334863066673, 0.012662945315241814, 0.010547308251261711, 0.010315663181245327, 0.022657567635178566, 0.0, 0.0], [0.8363587260246277, 0.013512525707483292, 0.01655869372189045, 0.008880337700247765, 0.05630703642964363, 0.0022557524498552084, 0.0023984003346413374, 0.0012933476828038692, 0.003150224220007658, 0.013540910556912422, 0.02165023609995842, 0.005392791703343391, 0.00432277237996459, 0.014378219842910767, 0.0], [0.7888551354408264, 0.014087573625147343, 0.022160783410072327, 0.04220619052648544, 0.06532405316829681, 0.0011144893942400813, 0.001956802560016513, 0.002514239400625229, 0.001448699738830328, 0.012026125565171242, 0.009607555344700813, 0.006895523052662611, 0.004557081963866949, 0.003263341262936592, 0.023982450366020203]]], "attentionHeadNames": ["L10H7", "L11H10", "L11H2"], "tokens": ["<|endoftext|>", "When", " John", " and", " Mary", " went", " to", " the", " shops", ",", " John", " gave", " the", " bag", " to"]}
     )
@@ -1306,9 +1306,9 @@ <h2>Residual Stream<a class="headerlink" href="#Residual-Stream" title="Permalin
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="4bc9653c-0aeb-4930-9e54-0fc92a4e0591" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("4bc9653c-0aeb-4930-9e54-0fc92a4e0591")) {                    Plotly.newPlot(                        "4bc9653c-0aeb-4930-9e54-0fc92a4e0591",                        [{"coloraxis":"coloraxis","name":"0","x":["\u003c|endoftext|\u003e_0","When_1"," John_2"," and_3"," Mary_4"," went_5"," to_6"," the_7"," shops_8",",_9"," John_10"," gave_11"," the_12"," bag_13"," to_14"],"z":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.00065016746521,-0.00024725322145968676,9.061812306754291e-06,-0.00036435198853723705,-4.832966806134209e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0010517835617065,-2.6816253011929803e-05,-2.0540108380373567e-05,-0.0004592325130943209,-0.0005939850234426558],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0002665519714355,0.0008679538150317967,0.0005159862921573222,-0.0009933760156854987,-0.0008652352844364941],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9949080944061279,0.005429603159427643,0.0016055518062785268,-0.0006179149495437741,-0.0016324687749147415],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9675664305686951,0.03134222328662872,0.0028418514411896467,-0.0012303927214816213,-0.0009862943552434444],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9675208926200867,0.031000729650259018,0.001782458508387208,-0.0004856795712839812,-0.000646778498776257],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9228320121765137,0.05134553834795952,0.004729225765913725,0.0009345413418486714,0.017047081142663956],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.6565485000610352,0.02385673113167286,0.002357447287067771,-1.7318130630883388e-05,0.3186914026737213],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.027302434667944908,0.03142485395073891,0.0018206859240308404,0.0007993190083652735,0.9383869171142578],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.026841893792152405,0.020981015637516975,0.0012513356050476432,0.0003238087520003319,1.0048280954360962],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.005687932018190622,0.014263695105910301,0.0004871227720286697,-8.984619489638135e-05,0.9914218783378601]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Position: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Position"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Patched Residual Stream"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="aa440a00-8bb4-44ef-8d9b-a548c3b23622" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("aa440a00-8bb4-44ef-8d9b-a548c3b23622")) {                    Plotly.newPlot(                        "aa440a00-8bb4-44ef-8d9b-a548c3b23622",                        [{"coloraxis":"coloraxis","name":"0","x":["\u003c|endoftext|\u003e_0","When_1"," John_2"," and_3"," Mary_4"," went_5"," to_6"," the_7"," shops_8",",_9"," John_10"," gave_11"," the_12"," bag_13"," to_14"],"z":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.00065016746521,-0.00024725322145968676,9.061812306754291e-06,-0.00036435198853723705,-4.832966806134209e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0010517835617065,-2.6816253011929803e-05,-2.0540108380373567e-05,-0.0004592325130943209,-0.0005939850234426558],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0002665519714355,0.0008679538150317967,0.0005159862921573222,-0.0009933760156854987,-0.0008652352844364941],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9949080944061279,0.005429603159427643,0.0016055518062785268,-0.0006179149495437741,-0.0016324687749147415],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9675664305686951,0.03134222328662872,0.0028418514411896467,-0.0012303927214816213,-0.0009862943552434444],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9675208926200867,0.031000729650259018,0.001782458508387208,-0.0004856795712839812,-0.000646778498776257],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9228320121765137,0.05134553834795952,0.004729225765913725,0.0009345413418486714,0.017047081142663956],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.6565485000610352,0.02385673113167286,0.002357447287067771,-1.7318130630883388e-05,0.3186914026737213],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.027302434667944908,0.03142485395073891,0.0018206859240308404,0.0007993190083652735,0.9383869171142578],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.026841893792152405,0.020981015637516975,0.0012513356050476432,0.0003238087520003319,1.0048280954360962],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.005687932018190622,0.014263695105910301,0.0004871227720286697,-8.984619489638135e-05,0.9914218783378601]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Position: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Position"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Patched Residual Stream"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('4bc9653c-0aeb-4930-9e54-0fc92a4e0591');
+var gd = document.getElementById('aa440a00-8bb4-44ef-8d9b-a548c3b23622');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -1394,9 +1394,9 @@ <h2>Layers<a class="headerlink" href="#Layers" title="Permalink to this heading"
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="191ca322-bcdc-49ad-a055-e5a33b6bafa3" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("191ca322-bcdc-49ad-a055-e5a33b6bafa3")) {                    Plotly.newPlot(                        "191ca322-bcdc-49ad-a055-e5a33b6bafa3",                        [{"coloraxis":"coloraxis","name":"0","x":["\u003c|endoftext|\u003e_0","When_1"," John_2"," and_3"," Mary_4"," went_5"," to_6"," the_7"," shops_8",",_9"," John_10"," gave_11"," the_12"," bag_13"," to_14"],"z":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.03545692563056946,-0.00024725322145968676,9.061812306754291e-06,-0.00036435198853723705,-4.832966806134209e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0029837191104888916,7.950901635922492e-05,2.097641845466569e-05,8.054944191826507e-05,-0.0005966364406049252],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0019127808045595884,0.000666748033836484,0.00039496077806688845,-0.0007048076367937028,-0.0002728612453211099],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.15463140606880188,0.0038022694643586874,0.0005171945667825639,-0.00012015292304567993,-0.0005605234182439744],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.005405941512435675,0.019581804051995277,0.0010072707664221525,-0.00024272232258226722,0.0007941504009068012],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.35209786891937256,0.0010532510932534933,0.00022453156998381019,0.00013320863945409656,8.259674359578639e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.11985944956541061,0.02124394290149212,0.002728075487539172,0.0013410811079666018,0.017973870038986206],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.013310795649886131,0.011509508825838566,0.00037489054375328124,-4.121446545468643e-05,0.2976022958755493],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.001500166254118085,0.017352566123008728,0.0005847889697179198,0.0010113989701494575,0.5697317123413086],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0001285099278902635,0.006301181390881538,0.00014156564429868013,0.00031266608857549727,0.2715233862400055],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0009370585321448743,8.679202437633649e-05,0.00033203151542693377,1.543864300401765e-06,-0.1929759979248047],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.4061770737171173]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Position: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Position"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Patched Attention Layer"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="d488530c-df0a-4dd0-a5d4-bd55b79cf332" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("d488530c-df0a-4dd0-a5d4-bd55b79cf332")) {                    Plotly.newPlot(                        "d488530c-df0a-4dd0-a5d4-bd55b79cf332",                        [{"coloraxis":"coloraxis","name":"0","x":["\u003c|endoftext|\u003e_0","When_1"," John_2"," and_3"," Mary_4"," went_5"," to_6"," the_7"," shops_8",",_9"," John_10"," gave_11"," the_12"," bag_13"," to_14"],"z":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.03545692563056946,-0.00024725322145968676,9.061812306754291e-06,-0.00036435198853723705,-4.832966806134209e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0029837191104888916,7.950901635922492e-05,2.097641845466569e-05,8.054944191826507e-05,-0.0005966364406049252],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0019127808045595884,0.000666748033836484,0.00039496077806688845,-0.0007048076367937028,-0.0002728612453211099],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.15463140606880188,0.0038022694643586874,0.0005171945667825639,-0.00012015292304567993,-0.0005605234182439744],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.005405941512435675,0.019581804051995277,0.0010072707664221525,-0.00024272232258226722,0.0007941504009068012],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.35209786891937256,0.0010532510932534933,0.00022453156998381019,0.00013320863945409656,8.259674359578639e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.11985944956541061,0.02124394290149212,0.002728075487539172,0.0013410811079666018,0.017973870038986206],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.013310795649886131,0.011509508825838566,0.00037489054375328124,-4.121446545468643e-05,0.2976022958755493],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.001500166254118085,0.017352566123008728,0.0005847889697179198,0.0010113989701494575,0.5697317123413086],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0001285099278902635,0.006301181390881538,0.00014156564429868013,0.00031266608857549727,0.2715233862400055],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0009370585321448743,8.679202437633649e-05,0.00033203151542693377,1.543864300401765e-06,-0.1929759979248047],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.4061770737171173]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Position: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Position"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Patched Attention Layer"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('191ca322-bcdc-49ad-a055-e5a33b6bafa3');
+var gd = document.getElementById('d488530c-df0a-4dd0-a5d4-bd55b79cf332');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -1443,9 +1443,9 @@ <h2>Layers<a class="headerlink" href="#Layers" title="Permalink to this heading"
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="53f8687c-322d-426d-8a84-d3e680bc0251" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("53f8687c-322d-426d-8a84-d3e680bc0251")) {                    Plotly.newPlot(                        "53f8687c-322d-426d-8a84-d3e680bc0251",                        [{"coloraxis":"coloraxis","name":"0","x":["\u003c|endoftext|\u003e_0","When_1"," John_2"," and_3"," Mary_4"," went_5"," to_6"," the_7"," shops_8",",_9"," John_10"," gave_11"," the_12"," bag_13"," to_14"],"z":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.8507904410362244,-0.00027849970501847565,-7.350136729655787e-05,-0.00047316084965132177,3.9704162190901116e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.008864667266607285,0.00022147741401568055,0.00014985552115831524,-4.8866662837099284e-05,0.000304174842312932],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.013550027273595333,5.839834557264112e-05,-0.0003300177922938019,-0.0006387235480360687,0.0007728383643552661],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0019474170403555036,0.0004988359869457781,0.00017297992599196732,0.00016838189912959933,0.0004079157952219248],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.019786769524216652,0.004128897562623024,-4.849747710977681e-05,-0.00016989219875540584,0.0007919353083707392],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.09652478247880936,-0.001882205600850284,-0.00048380010412074625,0.000710244697984308,-0.00018361916590947658],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.015899553894996643,-0.0008504678844474256,0.00012357627565506846,2.8863551051472314e-05,-0.007237736601382494],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.01036097388714552,0.003150892909616232,0.0005310557899065316,0.00023543930728919804,0.008496489375829697],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.012532956898212433,2.2285346858552657e-05,-0.0003539476892910898,8.638927829451859e-05,-0.02163148671388626],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.00033434730721637607,0.0008097903919406235,1.6344823961844668e-05,0.00012938254803884774,0.031624484807252884],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0013600102392956614,-0.0001945940311998129,-9.914294059854001e-05,-0.00014133071817923337,0.02876460924744606],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.020449254661798477]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Position: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Position"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Patched MLP Layer"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="11229a77-9a7e-45c1-802c-f409a3943559" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("11229a77-9a7e-45c1-802c-f409a3943559")) {                    Plotly.newPlot(                        "11229a77-9a7e-45c1-802c-f409a3943559",                        [{"coloraxis":"coloraxis","name":"0","x":["\u003c|endoftext|\u003e_0","When_1"," John_2"," and_3"," Mary_4"," went_5"," to_6"," the_7"," shops_8",",_9"," John_10"," gave_11"," the_12"," bag_13"," to_14"],"z":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.8507904410362244,-0.00027849970501847565,-7.350136729655787e-05,-0.00047316084965132177,3.9704162190901116e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.008864667266607285,0.00022147741401568055,0.00014985552115831524,-4.8866662837099284e-05,0.000304174842312932],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.013550027273595333,5.839834557264112e-05,-0.0003300177922938019,-0.0006387235480360687,0.0007728383643552661],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0019474170403555036,0.0004988359869457781,0.00017297992599196732,0.00016838189912959933,0.0004079157952219248],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.019786769524216652,0.004128897562623024,-4.849747710977681e-05,-0.00016989219875540584,0.0007919353083707392],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.09652478247880936,-0.001882205600850284,-0.00048380010412074625,0.000710244697984308,-0.00018361916590947658],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.015899553894996643,-0.0008504678844474256,0.00012357627565506846,2.8863551051472314e-05,-0.007237736601382494],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.01036097388714552,0.003150892909616232,0.0005310557899065316,0.00023543930728919804,0.008496489375829697],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.012532956898212433,2.2285346858552657e-05,-0.0003539476892910898,8.638927829451859e-05,-0.02163148671388626],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.00033434730721637607,0.0008097903919406235,1.6344823961844668e-05,0.00012938254803884774,0.031624484807252884],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0013600102392956614,-0.0001945940311998129,-9.914294059854001e-05,-0.00014133071817923337,0.02876460924744606],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.020449254661798477]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Position: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Position"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Patched MLP Layer"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('53f8687c-322d-426d-8a84-d3e680bc0251');
+var gd = document.getElementById('11229a77-9a7e-45c1-802c-f409a3943559');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -1526,9 +1526,9 @@ <h2>Heads<a class="headerlink" href="#Heads" title="Permalink to this heading">#
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="d7db9d56-c782-4460-85c8-5165d9b90be4" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("d7db9d56-c782-4460-85c8-5165d9b90be4")) {                    Plotly.newPlot(                        "d7db9d56-c782-4460-85c8-5165d9b90be4",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.000949342327658087,0.016124119982123375,0.0018556242575868964,0.0034389242064207792,-0.009823138825595379,0.011059673503041267,-0.0040638200007379055,-0.0015785341383889318,-0.001208073808811605,0.003828749991953373,-0.004257441032677889,-0.0011428623693063855],[-0.001076644053682685,-0.0003779782564379275,3.0206042538338806e-06,-0.000260510336374864,-0.00014149851631373167,0.0038328110240399837,-0.0004281202855054289,-0.0014291484840214252,-0.0009217541082762182,0.0006938663427717984,0.00043359093251638114,-0.0035708239302039146],[-0.0004966544220224023,0.0008070718613453209,0.0005420306115411222,-0.0005315592279657722,-0.0007152119651436806,-0.0010380810126662254,-0.0009492752142250538,-8.538240945199504e-05,0.0002774928288999945,0.0021076432894915342,-0.0001976146304514259,-0.0016398524167016149],[0.11626167595386505,0.00025017315056174994,-0.0014669395750388503,-0.0003969745012000203,0.01896214485168457,-0.00018882131553255022,0.011170395649969578,-0.0013304082676768303,-0.0007363561890088022,-0.00030310085276141763,-0.00014579448907170445,-0.0002225178322987631],[-0.0016502231592312455,0.00029299859306775033,-0.0014352232683449984,0.030841777101159096,-0.007431894540786743,-0.0002820908557623625,0.006017110776156187,-0.011006947606801987,-0.0012651296565309167,0.001490030437707901,-0.00018016225658357143,0.0029450219590216875],[-0.004210856277495623,0.0029608632903546095,0.002045620232820511,0.0013410139363259077,-0.0012189144035801291,0.34349939227104187,0.0005627049831673503,-0.0001257913827430457,-0.005152345169335604,0.016241183504462242,0.017089638859033585,-0.004173669498413801],[0.039775215089321136,0.015227536670863628,-0.001023011514917016,0.0008081123232841492,-0.004935599863529205,-0.002123551908880472,-0.014273260720074177,0.0013749118661507964,0.0014838549541309476,0.1302708089351654,-0.00033538774005137384,0.0012914760736748576],[0.0003724069392774254,0.0195147804915905,0.00022308839834295213,0.12424899637699127,-0.0004041232750751078,-0.007652163505554199,0.0013012762647122145,-0.0011244030902162194,-0.007449581753462553,0.19224178791046143,-0.0032753082923591137,-0.000501957256346941],[-0.001007740618661046,3.1212908652378246e-05,-0.0008586570620536804,0.012360110878944397,-0.00040429108776152134,-0.004328995477408171,0.31855517625808716,0.002329523442313075,0.002118081087246537,0.0001410957775078714,0.27793484926223755,0.005738543812185526],[0.0058897752314805984,-0.000968372100032866,0.009125681594014168,0.020675498992204666,-0.03700448200106621,0.014263628050684929,-0.04828385263681412,0.05834108963608742,0.0006516450084745884,0.2636081576347351,0.0004918886115774512,-0.0026110773906111717],[0.08374276012182236,0.020676370710134506,-0.0037437702994793653,0.01085111778229475,-0.0010960765648633242,0.0004738992138300091,0.04817944020032883,-0.47990989685058594,0.00018412260396871716,0.011861777864396572,0.06088537722826004,0.0008459034143015742],[0.005328110884875059,-0.011493667028844357,-0.11350765824317932,0.006330112461000681,0.0003166264505125582,-0.0011600126745179296,-0.022669030353426933,0.004070901777595282,0.007316238712519407,-0.008345123380422592,-0.27817434072494507,0.003635196480900049]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Patched Head Output"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="c9735339-8c98-45e1-988a-a5a0017e244e" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("c9735339-8c98-45e1-988a-a5a0017e244e")) {                    Plotly.newPlot(                        "c9735339-8c98-45e1-988a-a5a0017e244e",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.000949342327658087,0.016124119982123375,0.0018556242575868964,0.0034389242064207792,-0.009823138825595379,0.011059673503041267,-0.0040638200007379055,-0.0015785341383889318,-0.001208073808811605,0.003828749991953373,-0.004257441032677889,-0.0011428623693063855],[-0.001076644053682685,-0.0003779782564379275,3.0206042538338806e-06,-0.000260510336374864,-0.00014149851631373167,0.0038328110240399837,-0.0004281202855054289,-0.0014291484840214252,-0.0009217541082762182,0.0006938663427717984,0.00043359093251638114,-0.0035708239302039146],[-0.0004966544220224023,0.0008070718613453209,0.0005420306115411222,-0.0005315592279657722,-0.0007152119651436806,-0.0010380810126662254,-0.0009492752142250538,-8.538240945199504e-05,0.0002774928288999945,0.0021076432894915342,-0.0001976146304514259,-0.0016398524167016149],[0.11626167595386505,0.00025017315056174994,-0.0014669395750388503,-0.0003969745012000203,0.01896214485168457,-0.00018882131553255022,0.011170395649969578,-0.0013304082676768303,-0.0007363561890088022,-0.00030310085276141763,-0.00014579448907170445,-0.0002225178322987631],[-0.0016502231592312455,0.00029299859306775033,-0.0014352232683449984,0.030841777101159096,-0.007431894540786743,-0.0002820908557623625,0.006017110776156187,-0.011006947606801987,-0.0012651296565309167,0.001490030437707901,-0.00018016225658357143,0.0029450219590216875],[-0.004210856277495623,0.0029608632903546095,0.002045620232820511,0.0013410139363259077,-0.0012189144035801291,0.34349939227104187,0.0005627049831673503,-0.0001257913827430457,-0.005152345169335604,0.016241183504462242,0.017089638859033585,-0.004173669498413801],[0.039775215089321136,0.015227536670863628,-0.001023011514917016,0.0008081123232841492,-0.004935599863529205,-0.002123551908880472,-0.014273260720074177,0.0013749118661507964,0.0014838549541309476,0.1302708089351654,-0.00033538774005137384,0.0012914760736748576],[0.0003724069392774254,0.0195147804915905,0.00022308839834295213,0.12424899637699127,-0.0004041232750751078,-0.007652163505554199,0.0013012762647122145,-0.0011244030902162194,-0.007449581753462553,0.19224178791046143,-0.0032753082923591137,-0.000501957256346941],[-0.001007740618661046,3.1212908652378246e-05,-0.0008586570620536804,0.012360110878944397,-0.00040429108776152134,-0.004328995477408171,0.31855517625808716,0.002329523442313075,0.002118081087246537,0.0001410957775078714,0.27793484926223755,0.005738543812185526],[0.0058897752314805984,-0.000968372100032866,0.009125681594014168,0.020675498992204666,-0.03700448200106621,0.014263628050684929,-0.04828385263681412,0.05834108963608742,0.0006516450084745884,0.2636081576347351,0.0004918886115774512,-0.0026110773906111717],[0.08374276012182236,0.020676370710134506,-0.0037437702994793653,0.01085111778229475,-0.0010960765648633242,0.0004738992138300091,0.04817944020032883,-0.47990989685058594,0.00018412260396871716,0.011861777864396572,0.06088537722826004,0.0008459034143015742],[0.005328110884875059,-0.011493667028844357,-0.11350765824317932,0.006330112461000681,0.0003166264505125582,-0.0011600126745179296,-0.022669030353426933,0.004070901777595282,0.007316238712519407,-0.008345123380422592,-0.27817434072494507,0.003635196480900049]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Patched Head Output"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('d7db9d56-c782-4460-85c8-5165d9b90be4');
+var gd = document.getElementById('c9735339-8c98-45e1-988a-a5a0017e244e');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -1599,9 +1599,9 @@ <h2>Decomposing Heads<a class="headerlink" href="#Decomposing-Heads" title="Perm
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="b78bb017-f4f8-4418-b905-eac4d9764ea8" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("b78bb017-f4f8-4418-b905-eac4d9764ea8")) {                    Plotly.newPlot(                        "b78bb017-f4f8-4418-b905-eac4d9764ea8",                        [{"coloraxis":"coloraxis","name":"0","z":[[-0.00019808451179414988,0.005339857656508684,0.0006520813331007957,0.0035054110921919346,-0.008983243256807327,0.003481481224298477,-0.0008612077799625695,-3.356226807227358e-05,0.0005169931682758033,0.000442283577285707,-0.003907017409801483,-0.0001884857047116384],[-0.0004399341996759176,-0.00044604254071600735,-6.719166412949562e-05,7.296437252080068e-05,-3.587806349969469e-05,-0.001932314014993608,-0.0001571049797348678,1.550576780573465e-05,0.00020597163529600948,0.0003376699751242995,0.00035139694227837026,-0.0005663632764481008],[0.0002103011793224141,-0.0007198435487225652,0.00048675358993932605,-0.0005971734644845128,-0.0005919041577726603,-0.0005438430234789848,-0.0002275186125189066,-0.00047960481606423855,0.00020617301925085485,0.0011842446401715279,-0.00035743816988542676,-0.0009093025582842529],[0.0010397254955023527,-0.0001205221051350236,-7.69247199059464e-05,-0.0007269251509569585,-0.001310002407990396,-0.002310896525159478,0.010987145826220512,-5.06454634887632e-05,0.00014364650996867567,0.00015129870735108852,-7.920695497887209e-05,-1.9667490050778724e-05],[-0.0005367277772165835,-0.0008130795322358608,-0.000133275767439045,0.030610064044594765,-0.007185581140220165,0.00014941922563593835,0.001333831693045795,-0.011422313749790192,-0.0005329688428901136,0.0005127307958900928,0.00037368229823186994,0.002956030424684286],[7.4508234320092015e-06,6.712453796353657e-06,0.0015982352197170258,0.00033918028930202127,-0.0012586521916091442,-5.423662514658645e-05,0.0006328836898319423,-0.0002699077595025301,7.430685946019366e-05,-0.006704063154757023,0.0031767694745212793,-0.001728456816636026],[0.04863159358501434,0.015315738506615162,-0.0004640990518964827,-0.0001160583269665949,-4.839679240831174e-05,-0.003953501116484404,-0.017375119030475616,-0.0001535809424240142,0.0012203240767121315,-0.00017975950322579592,-0.0004269791825208813,0.0001233748917002231],[-2.849436532414984e-05,-0.001384309260174632,-0.00012176390737295151,0.13321569561958313,-0.00024480317370034754,-0.0073145609349012375,0.0003329376922920346,-0.000794418912846595,-0.007937678135931492,0.20841389894485474,-0.0001913720479933545,-0.0002066764427581802],[-0.0020480367820709944,-0.0003764008288271725,-0.0033146096393465996,-0.009665866382420063,-0.00031719700200483203,-0.005141303408890963,0.3171706795692444,0.002842522691935301,0.00047299303696490824,-0.0011535687372088432,0.27267059683799744,-0.00317572895437479],[-0.00043966571683995426,5.7257228036178276e-05,-0.002063408261165023,0.020066477358341217,-0.007870486006140709,0.01131565310060978,0.003057321300730109,0.06856381893157959,-0.0027472060173749924,-0.009278557263314724,0.0005067567108199,-0.0013152381870895624],[-0.012957250699400902,-0.00304490327835083,-0.017922621220350266,-0.004358463454991579,-0.0011523604625836015,0.0004998764488846064,-0.0031132360454648733,0.01958620175719261,4.3899446609430015e-05,0.012970171868801117,-0.007695660460740328,-0.0009143368806689978],[0.004100503865629435,-0.020459022372961044,-0.035875849425792694,0.014655501581728458,0.0008445944986306131,0.0017805789830163121,-0.018042270094156265,0.003519205143675208,0.008252425119280815,-0.0017664493061602116,0.044166937470436096,0.006474765483289957]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Patched Head Value"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="b42e697c-5002-48bc-8c51-ecb852488511" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("b42e697c-5002-48bc-8c51-ecb852488511")) {                    Plotly.newPlot(                        "b42e697c-5002-48bc-8c51-ecb852488511",                        [{"coloraxis":"coloraxis","name":"0","z":[[-0.00019808451179414988,0.005339857656508684,0.0006520813331007957,0.0035054110921919346,-0.008983243256807327,0.003481481224298477,-0.0008612077799625695,-3.356226807227358e-05,0.0005169931682758033,0.000442283577285707,-0.003907017409801483,-0.0001884857047116384],[-0.0004399341996759176,-0.00044604254071600735,-6.719166412949562e-05,7.296437252080068e-05,-3.587806349969469e-05,-0.001932314014993608,-0.0001571049797348678,1.550576780573465e-05,0.00020597163529600948,0.0003376699751242995,0.00035139694227837026,-0.0005663632764481008],[0.0002103011793224141,-0.0007198435487225652,0.00048675358993932605,-0.0005971734644845128,-0.0005919041577726603,-0.0005438430234789848,-0.0002275186125189066,-0.00047960481606423855,0.00020617301925085485,0.0011842446401715279,-0.00035743816988542676,-0.0009093025582842529],[0.0010397254955023527,-0.0001205221051350236,-7.69247199059464e-05,-0.0007269251509569585,-0.001310002407990396,-0.002310896525159478,0.010987145826220512,-5.06454634887632e-05,0.00014364650996867567,0.00015129870735108852,-7.920695497887209e-05,-1.9667490050778724e-05],[-0.0005367277772165835,-0.0008130795322358608,-0.000133275767439045,0.030610064044594765,-0.007185581140220165,0.00014941922563593835,0.001333831693045795,-0.011422313749790192,-0.0005329688428901136,0.0005127307958900928,0.00037368229823186994,0.002956030424684286],[7.4508234320092015e-06,6.712453796353657e-06,0.0015982352197170258,0.00033918028930202127,-0.0012586521916091442,-5.423662514658645e-05,0.0006328836898319423,-0.0002699077595025301,7.430685946019366e-05,-0.006704063154757023,0.0031767694745212793,-0.001728456816636026],[0.04863159358501434,0.015315738506615162,-0.0004640990518964827,-0.0001160583269665949,-4.839679240831174e-05,-0.003953501116484404,-0.017375119030475616,-0.0001535809424240142,0.0012203240767121315,-0.00017975950322579592,-0.0004269791825208813,0.0001233748917002231],[-2.849436532414984e-05,-0.001384309260174632,-0.00012176390737295151,0.13321569561958313,-0.00024480317370034754,-0.0073145609349012375,0.0003329376922920346,-0.000794418912846595,-0.007937678135931492,0.20841389894485474,-0.0001913720479933545,-0.0002066764427581802],[-0.0020480367820709944,-0.0003764008288271725,-0.0033146096393465996,-0.009665866382420063,-0.00031719700200483203,-0.005141303408890963,0.3171706795692444,0.002842522691935301,0.00047299303696490824,-0.0011535687372088432,0.27267059683799744,-0.00317572895437479],[-0.00043966571683995426,5.7257228036178276e-05,-0.002063408261165023,0.020066477358341217,-0.007870486006140709,0.01131565310060978,0.003057321300730109,0.06856381893157959,-0.0027472060173749924,-0.009278557263314724,0.0005067567108199,-0.0013152381870895624],[-0.012957250699400902,-0.00304490327835083,-0.017922621220350266,-0.004358463454991579,-0.0011523604625836015,0.0004998764488846064,-0.0031132360454648733,0.01958620175719261,4.3899446609430015e-05,0.012970171868801117,-0.007695660460740328,-0.0009143368806689978],[0.004100503865629435,-0.020459022372961044,-0.035875849425792694,0.014655501581728458,0.0008445944986306131,0.0017805789830163121,-0.018042270094156265,0.003519205143675208,0.008252425119280815,-0.0017664493061602116,0.044166937470436096,0.006474765483289957]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Patched Head Value"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('b78bb017-f4f8-4418-b905-eac4d9764ea8');
+var gd = document.getElementById('b42e697c-5002-48bc-8c51-ecb852488511');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -1655,9 +1655,9 @@ <h2>Decomposing Heads<a class="headerlink" href="#Decomposing-Heads" title="Perm
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="c2250b6e-3006-4a68-a458-2890b3c0d4df" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("c2250b6e-3006-4a68-a458-2890b3c0d4df")) {                    Plotly.newPlot(                        "c2250b6e-3006-4a68-a458-2890b3c0d4df",                        [{"hovertemplate":"\u003cb\u003e%{hovertext}\u003c\u002fb\u003e\u003cbr\u003e\u003cbr\u003eValue Patch=%{x}\u003cbr\u003eOutput Patch=%{y}\u003cbr\u003eLayer=%{marker.color}\u003cextra\u003e\u003c\u002fextra\u003e","hovertext":["L0H0","L0H1","L0H2","L0H3","L0H4","L0H5","L0H6","L0H7","L0H8","L0H9","L0H10","L0H11","L1H0","L1H1","L1H2","L1H3","L1H4","L1H5","L1H6","L1H7","L1H8","L1H9","L1H10","L1H11","L2H0","L2H1","L2H2","L2H3","L2H4","L2H5","L2H6","L2H7","L2H8","L2H9","L2H10","L2H11","L3H0","L3H1","L3H2","L3H3","L3H4","L3H5","L3H6","L3H7","L3H8","L3H9","L3H10","L3H11","L4H0","L4H1","L4H2","L4H3","L4H4","L4H5","L4H6","L4H7","L4H8","L4H9","L4H10","L4H11","L5H0","L5H1","L5H2","L5H3","L5H4","L5H5","L5H6","L5H7","L5H8","L5H9","L5H10","L5H11","L6H0","L6H1","L6H2","L6H3","L6H4","L6H5","L6H6","L6H7","L6H8","L6H9","L6H10","L6H11","L7H0","L7H1","L7H2","L7H3","L7H4","L7H5","L7H6","L7H7","L7H8","L7H9","L7H10","L7H11","L8H0","L8H1","L8H2","L8H3","L8H4","L8H5","L8H6","L8H7","L8H8","L8H9","L8H10","L8H11","L9H0","L9H1","L9H2","L9H3","L9H4","L9H5","L9H6","L9H7","L9H8","L9H9","L9H10","L9H11","L10H0","L10H1","L10H2","L10H3","L10H4","L10H5","L10H6","L10H7","L10H8","L10H9","L10H10","L10H11","L11H0","L11H1","L11H2","L11H3","L11H4","L11H5","L11H6","L11H7","L11H8","L11H9","L11H10","L11H11"],"legendgroup":"","marker":{"color":[0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7,7,7,7,7,8,8,8,8,8,8,8,8,8,8,8,8,9,9,9,9,9,9,9,9,9,9,9,9,10,10,10,10,10,10,10,10,10,10,10,10,11,11,11,11,11,11,11,11,11,11,11,11],"coloraxis":"coloraxis","symbol":"circle"},"mode":"markers","name":"","orientation":"v","showlegend":false,"x":[-0.00019808451179414988,0.005339857656508684,0.0006520813331007957,0.0035054110921919346,-0.008983243256807327,0.003481481224298477,-0.0008612077799625695,-3.356226807227358e-05,0.0005169931682758033,0.000442283577285707,-0.003907017409801483,-0.0001884857047116384,-0.0004399341996759176,-0.00044604254071600735,-6.719166412949562e-05,7.296437252080068e-05,-3.587806349969469e-05,-0.001932314014993608,-0.0001571049797348678,1.550576780573465e-05,0.00020597163529600948,0.0003376699751242995,0.00035139694227837026,-0.0005663632764481008,0.0002103011793224141,-0.0007198435487225652,0.00048675358993932605,-0.0005971734644845128,-0.0005919041577726603,-0.0005438430234789848,-0.0002275186125189066,-0.00047960481606423855,0.00020617301925085485,0.0011842446401715279,-0.00035743816988542676,-0.0009093025582842529,0.0010397254955023527,-0.0001205221051350236,-7.69247199059464e-05,-0.0007269251509569585,-0.001310002407990396,-0.002310896525159478,0.010987145826220512,-5.06454634887632e-05,0.00014364650996867567,0.00015129870735108852,-7.920695497887209e-05,-1.9667490050778724e-05,-0.0005367277772165835,-0.0008130795322358608,-0.000133275767439045,0.030610064044594765,-0.007185581140220165,0.00014941922563593835,0.001333831693045795,-0.011422313749790192,-0.0005329688428901136,0.0005127307958900928,0.00037368229823186994,0.002956030424684286,7.4508234320092015e-06,6.712453796353657e-06,0.0015982352197170258,0.00033918028930202127,-0.0012586521916091442,-5.423662514658645e-05,0.0006328836898319423,-0.0002699077595025301,7.430685946019366e-05,-0.006704063154757023,0.0031767694745212793,-0.001728456816636026,0.04863159358501434,0.015315738506615162,-0.0004640990518964827,-0.0001160583269665949,-4.839679240831174e-05,-0.003953501116484404,-0.017375119030475616,-0.0001535809424240142,0.0012203240767121315,-0.00017975950322579592,-0.0004269791825208813,0.0001233748917002231,-2.849436532414984e-05,-0.001384309260174632,-0.00012176390737295151,0.13321569561958313,-0.00024480317370034754,-0.0073145609349012375,0.0003329376922920346,-0.000794418912846595,-0.007937678135931492,0.20841389894485474,-0.0001913720479933545,-0.0002066764427581802,-0.0020480367820709944,-0.0003764008288271725,-0.0033146096393465996,-0.009665866382420063,-0.00031719700200483203,-0.005141303408890963,0.3171706795692444,0.002842522691935301,0.00047299303696490824,-0.0011535687372088432,0.27267059683799744,-0.00317572895437479,-0.00043966571683995426,5.7257228036178276e-05,-0.002063408261165023,0.020066477358341217,-0.007870486006140709,0.01131565310060978,0.003057321300730109,0.06856381893157959,-0.0027472060173749924,-0.009278557263314724,0.0005067567108199,-0.0013152381870895624,-0.012957250699400902,-0.00304490327835083,-0.017922621220350266,-0.004358463454991579,-0.0011523604625836015,0.0004998764488846064,-0.0031132360454648733,0.01958620175719261,4.3899446609430015e-05,0.012970171868801117,-0.007695660460740328,-0.0009143368806689978,0.004100503865629435,-0.020459022372961044,-0.035875849425792694,0.014655501581728458,0.0008445944986306131,0.0017805789830163121,-0.018042270094156265,0.003519205143675208,0.008252425119280815,-0.0017664493061602116,0.044166937470436096,0.006474765483289957],"xaxis":"x","y":[0.000949342327658087,0.016124119982123375,0.0018556242575868964,0.0034389242064207792,-0.009823138825595379,0.011059673503041267,-0.0040638200007379055,-0.0015785341383889318,-0.001208073808811605,0.003828749991953373,-0.004257441032677889,-0.0011428623693063855,-0.001076644053682685,-0.0003779782564379275,3.0206042538338806e-06,-0.000260510336374864,-0.00014149851631373167,0.0038328110240399837,-0.0004281202855054289,-0.0014291484840214252,-0.0009217541082762182,0.0006938663427717984,0.00043359093251638114,-0.0035708239302039146,-0.0004966544220224023,0.0008070718613453209,0.0005420306115411222,-0.0005315592279657722,-0.0007152119651436806,-0.0010380810126662254,-0.0009492752142250538,-8.538240945199504e-05,0.0002774928288999945,0.0021076432894915342,-0.0001976146304514259,-0.0016398524167016149,0.11626167595386505,0.00025017315056174994,-0.0014669395750388503,-0.0003969745012000203,0.01896214485168457,-0.00018882131553255022,0.011170395649969578,-0.0013304082676768303,-0.0007363561890088022,-0.00030310085276141763,-0.00014579448907170445,-0.0002225178322987631,-0.0016502231592312455,0.00029299859306775033,-0.0014352232683449984,0.030841777101159096,-0.007431894540786743,-0.0002820908557623625,0.006017110776156187,-0.011006947606801987,-0.0012651296565309167,0.001490030437707901,-0.00018016225658357143,0.0029450219590216875,-0.004210856277495623,0.0029608632903546095,0.002045620232820511,0.0013410139363259077,-0.0012189144035801291,0.34349939227104187,0.0005627049831673503,-0.0001257913827430457,-0.005152345169335604,0.016241183504462242,0.017089638859033585,-0.004173669498413801,0.039775215089321136,0.015227536670863628,-0.001023011514917016,0.0008081123232841492,-0.004935599863529205,-0.002123551908880472,-0.014273260720074177,0.0013749118661507964,0.0014838549541309476,0.1302708089351654,-0.00033538774005137384,0.0012914760736748576,0.0003724069392774254,0.0195147804915905,0.00022308839834295213,0.12424899637699127,-0.0004041232750751078,-0.007652163505554199,0.0013012762647122145,-0.0011244030902162194,-0.007449581753462553,0.19224178791046143,-0.0032753082923591137,-0.000501957256346941,-0.001007740618661046,3.1212908652378246e-05,-0.0008586570620536804,0.012360110878944397,-0.00040429108776152134,-0.004328995477408171,0.31855517625808716,0.002329523442313075,0.002118081087246537,0.0001410957775078714,0.27793484926223755,0.005738543812185526,0.0058897752314805984,-0.000968372100032866,0.009125681594014168,0.020675498992204666,-0.03700448200106621,0.014263628050684929,-0.04828385263681412,0.05834108963608742,0.0006516450084745884,0.2636081576347351,0.0004918886115774512,-0.0026110773906111717,0.08374276012182236,0.020676370710134506,-0.0037437702994793653,0.01085111778229475,-0.0010960765648633242,0.0004738992138300091,0.04817944020032883,-0.47990989685058594,0.00018412260396871716,0.011861777864396572,0.06088537722826004,0.0008459034143015742,0.005328110884875059,-0.011493667028844357,-0.11350765824317932,0.006330112461000681,0.0003166264505125582,-0.0011600126745179296,-0.022669030353426933,0.004070901777595282,0.007316238712519407,-0.008345123380422592,-0.27817434072494507,0.003635196480900049],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"Value Patch"},"range":[-0.5,0.5]},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"Output Patch"},"range":[-0.5,0.5]},"coloraxis":{"colorbar":{"title":{"text":"Layer"}},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]},"legend":{"tracegroupgap":0},"title":{"text":"Scatter plot of output patching vs value patching"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="40325163-a9c5-4067-9615-e2cea91c5832" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("40325163-a9c5-4067-9615-e2cea91c5832")) {                    Plotly.newPlot(                        "40325163-a9c5-4067-9615-e2cea91c5832",                        [{"hovertemplate":"\u003cb\u003e%{hovertext}\u003c\u002fb\u003e\u003cbr\u003e\u003cbr\u003eValue Patch=%{x}\u003cbr\u003eOutput Patch=%{y}\u003cbr\u003eLayer=%{marker.color}\u003cextra\u003e\u003c\u002fextra\u003e","hovertext":["L0H0","L0H1","L0H2","L0H3","L0H4","L0H5","L0H6","L0H7","L0H8","L0H9","L0H10","L0H11","L1H0","L1H1","L1H2","L1H3","L1H4","L1H5","L1H6","L1H7","L1H8","L1H9","L1H10","L1H11","L2H0","L2H1","L2H2","L2H3","L2H4","L2H5","L2H6","L2H7","L2H8","L2H9","L2H10","L2H11","L3H0","L3H1","L3H2","L3H3","L3H4","L3H5","L3H6","L3H7","L3H8","L3H9","L3H10","L3H11","L4H0","L4H1","L4H2","L4H3","L4H4","L4H5","L4H6","L4H7","L4H8","L4H9","L4H10","L4H11","L5H0","L5H1","L5H2","L5H3","L5H4","L5H5","L5H6","L5H7","L5H8","L5H9","L5H10","L5H11","L6H0","L6H1","L6H2","L6H3","L6H4","L6H5","L6H6","L6H7","L6H8","L6H9","L6H10","L6H11","L7H0","L7H1","L7H2","L7H3","L7H4","L7H5","L7H6","L7H7","L7H8","L7H9","L7H10","L7H11","L8H0","L8H1","L8H2","L8H3","L8H4","L8H5","L8H6","L8H7","L8H8","L8H9","L8H10","L8H11","L9H0","L9H1","L9H2","L9H3","L9H4","L9H5","L9H6","L9H7","L9H8","L9H9","L9H10","L9H11","L10H0","L10H1","L10H2","L10H3","L10H4","L10H5","L10H6","L10H7","L10H8","L10H9","L10H10","L10H11","L11H0","L11H1","L11H2","L11H3","L11H4","L11H5","L11H6","L11H7","L11H8","L11H9","L11H10","L11H11"],"legendgroup":"","marker":{"color":[0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7,7,7,7,7,8,8,8,8,8,8,8,8,8,8,8,8,9,9,9,9,9,9,9,9,9,9,9,9,10,10,10,10,10,10,10,10,10,10,10,10,11,11,11,11,11,11,11,11,11,11,11,11],"coloraxis":"coloraxis","symbol":"circle"},"mode":"markers","name":"","orientation":"v","showlegend":false,"x":[-0.00019808451179414988,0.005339857656508684,0.0006520813331007957,0.0035054110921919346,-0.008983243256807327,0.003481481224298477,-0.0008612077799625695,-3.356226807227358e-05,0.0005169931682758033,0.000442283577285707,-0.003907017409801483,-0.0001884857047116384,-0.0004399341996759176,-0.00044604254071600735,-6.719166412949562e-05,7.296437252080068e-05,-3.587806349969469e-05,-0.001932314014993608,-0.0001571049797348678,1.550576780573465e-05,0.00020597163529600948,0.0003376699751242995,0.00035139694227837026,-0.0005663632764481008,0.0002103011793224141,-0.0007198435487225652,0.00048675358993932605,-0.0005971734644845128,-0.0005919041577726603,-0.0005438430234789848,-0.0002275186125189066,-0.00047960481606423855,0.00020617301925085485,0.0011842446401715279,-0.00035743816988542676,-0.0009093025582842529,0.0010397254955023527,-0.0001205221051350236,-7.69247199059464e-05,-0.0007269251509569585,-0.001310002407990396,-0.002310896525159478,0.010987145826220512,-5.06454634887632e-05,0.00014364650996867567,0.00015129870735108852,-7.920695497887209e-05,-1.9667490050778724e-05,-0.0005367277772165835,-0.0008130795322358608,-0.000133275767439045,0.030610064044594765,-0.007185581140220165,0.00014941922563593835,0.001333831693045795,-0.011422313749790192,-0.0005329688428901136,0.0005127307958900928,0.00037368229823186994,0.002956030424684286,7.4508234320092015e-06,6.712453796353657e-06,0.0015982352197170258,0.00033918028930202127,-0.0012586521916091442,-5.423662514658645e-05,0.0006328836898319423,-0.0002699077595025301,7.430685946019366e-05,-0.006704063154757023,0.0031767694745212793,-0.001728456816636026,0.04863159358501434,0.015315738506615162,-0.0004640990518964827,-0.0001160583269665949,-4.839679240831174e-05,-0.003953501116484404,-0.017375119030475616,-0.0001535809424240142,0.0012203240767121315,-0.00017975950322579592,-0.0004269791825208813,0.0001233748917002231,-2.849436532414984e-05,-0.001384309260174632,-0.00012176390737295151,0.13321569561958313,-0.00024480317370034754,-0.0073145609349012375,0.0003329376922920346,-0.000794418912846595,-0.007937678135931492,0.20841389894485474,-0.0001913720479933545,-0.0002066764427581802,-0.0020480367820709944,-0.0003764008288271725,-0.0033146096393465996,-0.009665866382420063,-0.00031719700200483203,-0.005141303408890963,0.3171706795692444,0.002842522691935301,0.00047299303696490824,-0.0011535687372088432,0.27267059683799744,-0.00317572895437479,-0.00043966571683995426,5.7257228036178276e-05,-0.002063408261165023,0.020066477358341217,-0.007870486006140709,0.01131565310060978,0.003057321300730109,0.06856381893157959,-0.0027472060173749924,-0.009278557263314724,0.0005067567108199,-0.0013152381870895624,-0.012957250699400902,-0.00304490327835083,-0.017922621220350266,-0.004358463454991579,-0.0011523604625836015,0.0004998764488846064,-0.0031132360454648733,0.01958620175719261,4.3899446609430015e-05,0.012970171868801117,-0.007695660460740328,-0.0009143368806689978,0.004100503865629435,-0.020459022372961044,-0.035875849425792694,0.014655501581728458,0.0008445944986306131,0.0017805789830163121,-0.018042270094156265,0.003519205143675208,0.008252425119280815,-0.0017664493061602116,0.044166937470436096,0.006474765483289957],"xaxis":"x","y":[0.000949342327658087,0.016124119982123375,0.0018556242575868964,0.0034389242064207792,-0.009823138825595379,0.011059673503041267,-0.0040638200007379055,-0.0015785341383889318,-0.001208073808811605,0.003828749991953373,-0.004257441032677889,-0.0011428623693063855,-0.001076644053682685,-0.0003779782564379275,3.0206042538338806e-06,-0.000260510336374864,-0.00014149851631373167,0.0038328110240399837,-0.0004281202855054289,-0.0014291484840214252,-0.0009217541082762182,0.0006938663427717984,0.00043359093251638114,-0.0035708239302039146,-0.0004966544220224023,0.0008070718613453209,0.0005420306115411222,-0.0005315592279657722,-0.0007152119651436806,-0.0010380810126662254,-0.0009492752142250538,-8.538240945199504e-05,0.0002774928288999945,0.0021076432894915342,-0.0001976146304514259,-0.0016398524167016149,0.11626167595386505,0.00025017315056174994,-0.0014669395750388503,-0.0003969745012000203,0.01896214485168457,-0.00018882131553255022,0.011170395649969578,-0.0013304082676768303,-0.0007363561890088022,-0.00030310085276141763,-0.00014579448907170445,-0.0002225178322987631,-0.0016502231592312455,0.00029299859306775033,-0.0014352232683449984,0.030841777101159096,-0.007431894540786743,-0.0002820908557623625,0.006017110776156187,-0.011006947606801987,-0.0012651296565309167,0.001490030437707901,-0.00018016225658357143,0.0029450219590216875,-0.004210856277495623,0.0029608632903546095,0.002045620232820511,0.0013410139363259077,-0.0012189144035801291,0.34349939227104187,0.0005627049831673503,-0.0001257913827430457,-0.005152345169335604,0.016241183504462242,0.017089638859033585,-0.004173669498413801,0.039775215089321136,0.015227536670863628,-0.001023011514917016,0.0008081123232841492,-0.004935599863529205,-0.002123551908880472,-0.014273260720074177,0.0013749118661507964,0.0014838549541309476,0.1302708089351654,-0.00033538774005137384,0.0012914760736748576,0.0003724069392774254,0.0195147804915905,0.00022308839834295213,0.12424899637699127,-0.0004041232750751078,-0.007652163505554199,0.0013012762647122145,-0.0011244030902162194,-0.007449581753462553,0.19224178791046143,-0.0032753082923591137,-0.000501957256346941,-0.001007740618661046,3.1212908652378246e-05,-0.0008586570620536804,0.012360110878944397,-0.00040429108776152134,-0.004328995477408171,0.31855517625808716,0.002329523442313075,0.002118081087246537,0.0001410957775078714,0.27793484926223755,0.005738543812185526,0.0058897752314805984,-0.000968372100032866,0.009125681594014168,0.020675498992204666,-0.03700448200106621,0.014263628050684929,-0.04828385263681412,0.05834108963608742,0.0006516450084745884,0.2636081576347351,0.0004918886115774512,-0.0026110773906111717,0.08374276012182236,0.020676370710134506,-0.0037437702994793653,0.01085111778229475,-0.0010960765648633242,0.0004738992138300091,0.04817944020032883,-0.47990989685058594,0.00018412260396871716,0.011861777864396572,0.06088537722826004,0.0008459034143015742,0.005328110884875059,-0.011493667028844357,-0.11350765824317932,0.006330112461000681,0.0003166264505125582,-0.0011600126745179296,-0.022669030353426933,0.004070901777595282,0.007316238712519407,-0.008345123380422592,-0.27817434072494507,0.003635196480900049],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"Value Patch"},"range":[-0.5,0.5]},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"Output Patch"},"range":[-0.5,0.5]},"coloraxis":{"colorbar":{"title":{"text":"Layer"}},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]},"legend":{"tracegroupgap":0},"title":{"text":"Scatter plot of output patching vs value patching"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('c2250b6e-3006-4a68-a458-2890b3c0d4df');
+var gd = document.getElementById('40325163-a9c5-4067-9615-e2cea91c5832');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -1744,9 +1744,9 @@ <h2>Decomposing Heads<a class="headerlink" href="#Decomposing-Heads" title="Perm
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="f21b691b-6def-4049-8b21-6c417fc5a93c" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("f21b691b-6def-4049-8b21-6c417fc5a93c")) {                    Plotly.newPlot(                        "f21b691b-6def-4049-8b21-6c417fc5a93c",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.0006398981786333025,0.005319283809512854,0.0011588380439206958,-5.923740172875114e-05,-0.0010669109178707004,0.005080018658190966,-0.0030812176410108805,-0.0020524668507277966,-0.0014406939735636115,0.003492556745186448,-0.0025678491219878197,-0.0009152094717137516],[-0.0007601853576488793,0.0001696908293524757,0.00012250228610355407,-0.00034918184974230826,1.5405081285280176e-05,0.005008597858250141,-0.0002970260684378445,-0.0014443857362493873,-0.001099835499189794,0.00047436909517273307,5.108177356305532e-05,-0.003494771895930171],[-0.0007234682561829686,0.0017475537024438381,-0.0001551919267512858,5.749216506956145e-05,-9.699495421955362e-05,-0.00042355581535957754,-0.000791196885984391,0.0002723913639783859,0.00010212998313363642,0.00042187771759927273,0.00015183570212684572,-0.0007432028651237488],[0.11458060890436172,0.00021201284835115075,-0.000942025741096586,0.0004294627869967371,0.02004331909120083,0.002104991814121604,7.662265852559358e-05,-0.0015435622772201896,-0.0008476486545987427,-0.0005817683413624763,0.00011934742360608652,-1.9130493456032127e-05],[-0.001126819639466703,0.0012377764796838164,-0.0012320708483457565,-0.0005953610525466502,-0.0007538085337728262,-0.0005843526450917125,0.004813097883015871,0.00018163899949286133,-0.0005363250384107232,0.0008580865105614066,-0.00029840212664566934,-1.2183103535789996e-05],[-0.004240894690155983,0.0029526741709560156,0.0005220275488682091,0.0009532691328786314,0.0001625084987608716,0.3435087502002716,-0.0003038727736566216,0.00010364028275944293,-0.005300321150571108,0.024866117164492607,0.014383613131940365,-0.0023283488117158413],[-0.0023879888467490673,-0.002172317821532488,-0.00047651707427576184,0.0004340272571425885,-0.004673780873417854,0.001858678413555026,-0.0026543056592345238,0.0014364985981956124,0.000303067296044901,0.1304299533367157,8.957769750850275e-05,0.0011777335312217474],[0.0003194456803612411,0.020570717751979828,0.00031887509976513684,-0.002512404229491949,-0.0002621884341351688,-0.00024587716325186193,0.0005522335413843393,-0.00043134228326380253,0.0002570534124970436,0.008090788498520851,-0.003069269470870495,-0.0004241599526721984],[0.0009768297895789146,0.00039294702582992613,0.0017541655106469989,0.022596266120672226,-4.443644138518721e-05,0.00014159921556711197,0.00958488043397665,-0.0003154182049911469,0.0015266804257407784,0.0011816268088296056,-0.01077402476221323,0.009365785866975784],[0.006314304657280445,-0.001094431965611875,0.011661848053336143,0.0013478941982612014,-0.02918611839413643,0.0038340529426932335,-0.04409320279955864,-0.005031823180615902,0.004822427872568369,0.2766473889350891,-3.1280032999347895e-05,-0.0006619150517508388],[0.09538879990577698,0.025069067254662514,0.014238725416362286,0.01475451048463583,9.914294059854001e-05,-8.977906691143289e-05,0.0508294515311718,-0.5050979852676392,0.00014717054727952927,-0.0016020277980715036,0.06883121281862259,0.0023271406535059214],[0.0013427927624434233,0.009631095454096794,-0.07776330411434174,-0.007728920318186283,-0.0005721024353988469,-0.0029568022582679987,-0.00494667561724782,0.000458796217571944,-0.0006325145368464291,-0.006519974209368229,-0.32048866152763367,-0.002472163178026676]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Patched Head Pattern"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="857c4615-23ce-4e0c-b819-9a1aa3fd5753" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("857c4615-23ce-4e0c-b819-9a1aa3fd5753")) {                    Plotly.newPlot(                        "857c4615-23ce-4e0c-b819-9a1aa3fd5753",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.0006398981786333025,0.005319283809512854,0.0011588380439206958,-5.923740172875114e-05,-0.0010669109178707004,0.005080018658190966,-0.0030812176410108805,-0.0020524668507277966,-0.0014406939735636115,0.003492556745186448,-0.0025678491219878197,-0.0009152094717137516],[-0.0007601853576488793,0.0001696908293524757,0.00012250228610355407,-0.00034918184974230826,1.5405081285280176e-05,0.005008597858250141,-0.0002970260684378445,-0.0014443857362493873,-0.001099835499189794,0.00047436909517273307,5.108177356305532e-05,-0.003494771895930171],[-0.0007234682561829686,0.0017475537024438381,-0.0001551919267512858,5.749216506956145e-05,-9.699495421955362e-05,-0.00042355581535957754,-0.000791196885984391,0.0002723913639783859,0.00010212998313363642,0.00042187771759927273,0.00015183570212684572,-0.0007432028651237488],[0.11458060890436172,0.00021201284835115075,-0.000942025741096586,0.0004294627869967371,0.02004331909120083,0.002104991814121604,7.662265852559358e-05,-0.0015435622772201896,-0.0008476486545987427,-0.0005817683413624763,0.00011934742360608652,-1.9130493456032127e-05],[-0.001126819639466703,0.0012377764796838164,-0.0012320708483457565,-0.0005953610525466502,-0.0007538085337728262,-0.0005843526450917125,0.004813097883015871,0.00018163899949286133,-0.0005363250384107232,0.0008580865105614066,-0.00029840212664566934,-1.2183103535789996e-05],[-0.004240894690155983,0.0029526741709560156,0.0005220275488682091,0.0009532691328786314,0.0001625084987608716,0.3435087502002716,-0.0003038727736566216,0.00010364028275944293,-0.005300321150571108,0.024866117164492607,0.014383613131940365,-0.0023283488117158413],[-0.0023879888467490673,-0.002172317821532488,-0.00047651707427576184,0.0004340272571425885,-0.004673780873417854,0.001858678413555026,-0.0026543056592345238,0.0014364985981956124,0.000303067296044901,0.1304299533367157,8.957769750850275e-05,0.0011777335312217474],[0.0003194456803612411,0.020570717751979828,0.00031887509976513684,-0.002512404229491949,-0.0002621884341351688,-0.00024587716325186193,0.0005522335413843393,-0.00043134228326380253,0.0002570534124970436,0.008090788498520851,-0.003069269470870495,-0.0004241599526721984],[0.0009768297895789146,0.00039294702582992613,0.0017541655106469989,0.022596266120672226,-4.443644138518721e-05,0.00014159921556711197,0.00958488043397665,-0.0003154182049911469,0.0015266804257407784,0.0011816268088296056,-0.01077402476221323,0.009365785866975784],[0.006314304657280445,-0.001094431965611875,0.011661848053336143,0.0013478941982612014,-0.02918611839413643,0.0038340529426932335,-0.04409320279955864,-0.005031823180615902,0.004822427872568369,0.2766473889350891,-3.1280032999347895e-05,-0.0006619150517508388],[0.09538879990577698,0.025069067254662514,0.014238725416362286,0.01475451048463583,9.914294059854001e-05,-8.977906691143289e-05,0.0508294515311718,-0.5050979852676392,0.00014717054727952927,-0.0016020277980715036,0.06883121281862259,0.0023271406535059214],[0.0013427927624434233,0.009631095454096794,-0.07776330411434174,-0.007728920318186283,-0.0005721024353988469,-0.0029568022582679987,-0.00494667561724782,0.000458796217571944,-0.0006325145368464291,-0.006519974209368229,-0.32048866152763367,-0.002472163178026676]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Patched Head Pattern"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('f21b691b-6def-4049-8b21-6c417fc5a93c');
+var gd = document.getElementById('857c4615-23ce-4e0c-b819-9a1aa3fd5753');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -1774,9 +1774,9 @@ <h2>Decomposing Heads<a class="headerlink" href="#Decomposing-Heads" title="Perm
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="d6f23061-00f8-4a84-9d17-cede03fea051" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("d6f23061-00f8-4a84-9d17-cede03fea051")) {                    Plotly.newPlot(                        "d6f23061-00f8-4a84-9d17-cede03fea051",                        [{"hovertemplate":"\u003cb\u003e%{hovertext}\u003c\u002fb\u003e\u003cbr\u003e\u003cbr\u003eAttention Patch=%{x}\u003cbr\u003eOutput Patch=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","hovertext":["L0H0","L0H1","L0H2","L0H3","L0H4","L0H5","L0H6","L0H7","L0H8","L0H9","L0H10","L0H11","L1H0","L1H1","L1H2","L1H3","L1H4","L1H5","L1H6","L1H7","L1H8","L1H9","L1H10","L1H11","L2H0","L2H1","L2H2","L2H3","L2H4","L2H5","L2H6","L2H7","L2H8","L2H9","L2H10","L2H11","L3H0","L3H1","L3H2","L3H3","L3H4","L3H5","L3H6","L3H7","L3H8","L3H9","L3H10","L3H11","L4H0","L4H1","L4H2","L4H3","L4H4","L4H5","L4H6","L4H7","L4H8","L4H9","L4H10","L4H11","L5H0","L5H1","L5H2","L5H3","L5H4","L5H5","L5H6","L5H7","L5H8","L5H9","L5H10","L5H11","L6H0","L6H1","L6H2","L6H3","L6H4","L6H5","L6H6","L6H7","L6H8","L6H9","L6H10","L6H11","L7H0","L7H1","L7H2","L7H3","L7H4","L7H5","L7H6","L7H7","L7H8","L7H9","L7H10","L7H11","L8H0","L8H1","L8H2","L8H3","L8H4","L8H5","L8H6","L8H7","L8H8","L8H9","L8H10","L8H11","L9H0","L9H1","L9H2","L9H3","L9H4","L9H5","L9H6","L9H7","L9H8","L9H9","L9H10","L9H11","L10H0","L10H1","L10H2","L10H3","L10H4","L10H5","L10H6","L10H7","L10H8","L10H9","L10H10","L10H11","L11H0","L11H1","L11H2","L11H3","L11H4","L11H5","L11H6","L11H7","L11H8","L11H9","L11H10","L11H11"],"legendgroup":"","marker":{"color":"#636efa","symbol":"circle"},"mode":"markers","name":"","orientation":"v","showlegend":false,"x":[0.0006398981786333025,0.005319283809512854,0.0011588380439206958,-5.923740172875114e-05,-0.0010669109178707004,0.005080018658190966,-0.0030812176410108805,-0.0020524668507277966,-0.0014406939735636115,0.003492556745186448,-0.0025678491219878197,-0.0009152094717137516,-0.0007601853576488793,0.0001696908293524757,0.00012250228610355407,-0.00034918184974230826,1.5405081285280176e-05,0.005008597858250141,-0.0002970260684378445,-0.0014443857362493873,-0.001099835499189794,0.00047436909517273307,5.108177356305532e-05,-0.003494771895930171,-0.0007234682561829686,0.0017475537024438381,-0.0001551919267512858,5.749216506956145e-05,-9.699495421955362e-05,-0.00042355581535957754,-0.000791196885984391,0.0002723913639783859,0.00010212998313363642,0.00042187771759927273,0.00015183570212684572,-0.0007432028651237488,0.11458060890436172,0.00021201284835115075,-0.000942025741096586,0.0004294627869967371,0.02004331909120083,0.002104991814121604,7.662265852559358e-05,-0.0015435622772201896,-0.0008476486545987427,-0.0005817683413624763,0.00011934742360608652,-1.9130493456032127e-05,-0.001126819639466703,0.0012377764796838164,-0.0012320708483457565,-0.0005953610525466502,-0.0007538085337728262,-0.0005843526450917125,0.004813097883015871,0.00018163899949286133,-0.0005363250384107232,0.0008580865105614066,-0.00029840212664566934,-1.2183103535789996e-05,-0.004240894690155983,0.0029526741709560156,0.0005220275488682091,0.0009532691328786314,0.0001625084987608716,0.3435087502002716,-0.0003038727736566216,0.00010364028275944293,-0.005300321150571108,0.024866117164492607,0.014383613131940365,-0.0023283488117158413,-0.0023879888467490673,-0.002172317821532488,-0.00047651707427576184,0.0004340272571425885,-0.004673780873417854,0.001858678413555026,-0.0026543056592345238,0.0014364985981956124,0.000303067296044901,0.1304299533367157,8.957769750850275e-05,0.0011777335312217474,0.0003194456803612411,0.020570717751979828,0.00031887509976513684,-0.002512404229491949,-0.0002621884341351688,-0.00024587716325186193,0.0005522335413843393,-0.00043134228326380253,0.0002570534124970436,0.008090788498520851,-0.003069269470870495,-0.0004241599526721984,0.0009768297895789146,0.00039294702582992613,0.0017541655106469989,0.022596266120672226,-4.443644138518721e-05,0.00014159921556711197,0.00958488043397665,-0.0003154182049911469,0.0015266804257407784,0.0011816268088296056,-0.01077402476221323,0.009365785866975784,0.006314304657280445,-0.001094431965611875,0.011661848053336143,0.0013478941982612014,-0.02918611839413643,0.0038340529426932335,-0.04409320279955864,-0.005031823180615902,0.004822427872568369,0.2766473889350891,-3.1280032999347895e-05,-0.0006619150517508388,0.09538879990577698,0.025069067254662514,0.014238725416362286,0.01475451048463583,9.914294059854001e-05,-8.977906691143289e-05,0.0508294515311718,-0.5050979852676392,0.00014717054727952927,-0.0016020277980715036,0.06883121281862259,0.0023271406535059214,0.0013427927624434233,0.009631095454096794,-0.07776330411434174,-0.007728920318186283,-0.0005721024353988469,-0.0029568022582679987,-0.00494667561724782,0.000458796217571944,-0.0006325145368464291,-0.006519974209368229,-0.32048866152763367,-0.002472163178026676],"xaxis":"x","y":[0.000949342327658087,0.016124119982123375,0.0018556242575868964,0.0034389242064207792,-0.009823138825595379,0.011059673503041267,-0.0040638200007379055,-0.0015785341383889318,-0.001208073808811605,0.003828749991953373,-0.004257441032677889,-0.0011428623693063855,-0.001076644053682685,-0.0003779782564379275,3.0206042538338806e-06,-0.000260510336374864,-0.00014149851631373167,0.0038328110240399837,-0.0004281202855054289,-0.0014291484840214252,-0.0009217541082762182,0.0006938663427717984,0.00043359093251638114,-0.0035708239302039146,-0.0004966544220224023,0.0008070718613453209,0.0005420306115411222,-0.0005315592279657722,-0.0007152119651436806,-0.0010380810126662254,-0.0009492752142250538,-8.538240945199504e-05,0.0002774928288999945,0.0021076432894915342,-0.0001976146304514259,-0.0016398524167016149,0.11626167595386505,0.00025017315056174994,-0.0014669395750388503,-0.0003969745012000203,0.01896214485168457,-0.00018882131553255022,0.011170395649969578,-0.0013304082676768303,-0.0007363561890088022,-0.00030310085276141763,-0.00014579448907170445,-0.0002225178322987631,-0.0016502231592312455,0.00029299859306775033,-0.0014352232683449984,0.030841777101159096,-0.007431894540786743,-0.0002820908557623625,0.006017110776156187,-0.011006947606801987,-0.0012651296565309167,0.001490030437707901,-0.00018016225658357143,0.0029450219590216875,-0.004210856277495623,0.0029608632903546095,0.002045620232820511,0.0013410139363259077,-0.0012189144035801291,0.34349939227104187,0.0005627049831673503,-0.0001257913827430457,-0.005152345169335604,0.016241183504462242,0.017089638859033585,-0.004173669498413801,0.039775215089321136,0.015227536670863628,-0.001023011514917016,0.0008081123232841492,-0.004935599863529205,-0.002123551908880472,-0.014273260720074177,0.0013749118661507964,0.0014838549541309476,0.1302708089351654,-0.00033538774005137384,0.0012914760736748576,0.0003724069392774254,0.0195147804915905,0.00022308839834295213,0.12424899637699127,-0.0004041232750751078,-0.007652163505554199,0.0013012762647122145,-0.0011244030902162194,-0.007449581753462553,0.19224178791046143,-0.0032753082923591137,-0.000501957256346941,-0.001007740618661046,3.1212908652378246e-05,-0.0008586570620536804,0.012360110878944397,-0.00040429108776152134,-0.004328995477408171,0.31855517625808716,0.002329523442313075,0.002118081087246537,0.0001410957775078714,0.27793484926223755,0.005738543812185526,0.0058897752314805984,-0.000968372100032866,0.009125681594014168,0.020675498992204666,-0.03700448200106621,0.014263628050684929,-0.04828385263681412,0.05834108963608742,0.0006516450084745884,0.2636081576347351,0.0004918886115774512,-0.0026110773906111717,0.08374276012182236,0.020676370710134506,-0.0037437702994793653,0.01085111778229475,-0.0010960765648633242,0.0004738992138300091,0.04817944020032883,-0.47990989685058594,0.00018412260396871716,0.011861777864396572,0.06088537722826004,0.0008459034143015742,0.005328110884875059,-0.011493667028844357,-0.11350765824317932,0.006330112461000681,0.0003166264505125582,-0.0011600126745179296,-0.022669030353426933,0.004070901777595282,0.007316238712519407,-0.008345123380422592,-0.27817434072494507,0.003635196480900049],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"Attention Patch"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"Output Patch"}},"legend":{"tracegroupgap":0},"title":{"text":"Scatter plot of output patching vs attention patching"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="a4bdc2e4-754f-45ec-9243-67a82b72f8a2" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("a4bdc2e4-754f-45ec-9243-67a82b72f8a2")) {                    Plotly.newPlot(                        "a4bdc2e4-754f-45ec-9243-67a82b72f8a2",                        [{"hovertemplate":"\u003cb\u003e%{hovertext}\u003c\u002fb\u003e\u003cbr\u003e\u003cbr\u003eAttention Patch=%{x}\u003cbr\u003eOutput Patch=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","hovertext":["L0H0","L0H1","L0H2","L0H3","L0H4","L0H5","L0H6","L0H7","L0H8","L0H9","L0H10","L0H11","L1H0","L1H1","L1H2","L1H3","L1H4","L1H5","L1H6","L1H7","L1H8","L1H9","L1H10","L1H11","L2H0","L2H1","L2H2","L2H3","L2H4","L2H5","L2H6","L2H7","L2H8","L2H9","L2H10","L2H11","L3H0","L3H1","L3H2","L3H3","L3H4","L3H5","L3H6","L3H7","L3H8","L3H9","L3H10","L3H11","L4H0","L4H1","L4H2","L4H3","L4H4","L4H5","L4H6","L4H7","L4H8","L4H9","L4H10","L4H11","L5H0","L5H1","L5H2","L5H3","L5H4","L5H5","L5H6","L5H7","L5H8","L5H9","L5H10","L5H11","L6H0","L6H1","L6H2","L6H3","L6H4","L6H5","L6H6","L6H7","L6H8","L6H9","L6H10","L6H11","L7H0","L7H1","L7H2","L7H3","L7H4","L7H5","L7H6","L7H7","L7H8","L7H9","L7H10","L7H11","L8H0","L8H1","L8H2","L8H3","L8H4","L8H5","L8H6","L8H7","L8H8","L8H9","L8H10","L8H11","L9H0","L9H1","L9H2","L9H3","L9H4","L9H5","L9H6","L9H7","L9H8","L9H9","L9H10","L9H11","L10H0","L10H1","L10H2","L10H3","L10H4","L10H5","L10H6","L10H7","L10H8","L10H9","L10H10","L10H11","L11H0","L11H1","L11H2","L11H3","L11H4","L11H5","L11H6","L11H7","L11H8","L11H9","L11H10","L11H11"],"legendgroup":"","marker":{"color":"#636efa","symbol":"circle"},"mode":"markers","name":"","orientation":"v","showlegend":false,"x":[0.0006398981786333025,0.005319283809512854,0.0011588380439206958,-5.923740172875114e-05,-0.0010669109178707004,0.005080018658190966,-0.0030812176410108805,-0.0020524668507277966,-0.0014406939735636115,0.003492556745186448,-0.0025678491219878197,-0.0009152094717137516,-0.0007601853576488793,0.0001696908293524757,0.00012250228610355407,-0.00034918184974230826,1.5405081285280176e-05,0.005008597858250141,-0.0002970260684378445,-0.0014443857362493873,-0.001099835499189794,0.00047436909517273307,5.108177356305532e-05,-0.003494771895930171,-0.0007234682561829686,0.0017475537024438381,-0.0001551919267512858,5.749216506956145e-05,-9.699495421955362e-05,-0.00042355581535957754,-0.000791196885984391,0.0002723913639783859,0.00010212998313363642,0.00042187771759927273,0.00015183570212684572,-0.0007432028651237488,0.11458060890436172,0.00021201284835115075,-0.000942025741096586,0.0004294627869967371,0.02004331909120083,0.002104991814121604,7.662265852559358e-05,-0.0015435622772201896,-0.0008476486545987427,-0.0005817683413624763,0.00011934742360608652,-1.9130493456032127e-05,-0.001126819639466703,0.0012377764796838164,-0.0012320708483457565,-0.0005953610525466502,-0.0007538085337728262,-0.0005843526450917125,0.004813097883015871,0.00018163899949286133,-0.0005363250384107232,0.0008580865105614066,-0.00029840212664566934,-1.2183103535789996e-05,-0.004240894690155983,0.0029526741709560156,0.0005220275488682091,0.0009532691328786314,0.0001625084987608716,0.3435087502002716,-0.0003038727736566216,0.00010364028275944293,-0.005300321150571108,0.024866117164492607,0.014383613131940365,-0.0023283488117158413,-0.0023879888467490673,-0.002172317821532488,-0.00047651707427576184,0.0004340272571425885,-0.004673780873417854,0.001858678413555026,-0.0026543056592345238,0.0014364985981956124,0.000303067296044901,0.1304299533367157,8.957769750850275e-05,0.0011777335312217474,0.0003194456803612411,0.020570717751979828,0.00031887509976513684,-0.002512404229491949,-0.0002621884341351688,-0.00024587716325186193,0.0005522335413843393,-0.00043134228326380253,0.0002570534124970436,0.008090788498520851,-0.003069269470870495,-0.0004241599526721984,0.0009768297895789146,0.00039294702582992613,0.0017541655106469989,0.022596266120672226,-4.443644138518721e-05,0.00014159921556711197,0.00958488043397665,-0.0003154182049911469,0.0015266804257407784,0.0011816268088296056,-0.01077402476221323,0.009365785866975784,0.006314304657280445,-0.001094431965611875,0.011661848053336143,0.0013478941982612014,-0.02918611839413643,0.0038340529426932335,-0.04409320279955864,-0.005031823180615902,0.004822427872568369,0.2766473889350891,-3.1280032999347895e-05,-0.0006619150517508388,0.09538879990577698,0.025069067254662514,0.014238725416362286,0.01475451048463583,9.914294059854001e-05,-8.977906691143289e-05,0.0508294515311718,-0.5050979852676392,0.00014717054727952927,-0.0016020277980715036,0.06883121281862259,0.0023271406535059214,0.0013427927624434233,0.009631095454096794,-0.07776330411434174,-0.007728920318186283,-0.0005721024353988469,-0.0029568022582679987,-0.00494667561724782,0.000458796217571944,-0.0006325145368464291,-0.006519974209368229,-0.32048866152763367,-0.002472163178026676],"xaxis":"x","y":[0.000949342327658087,0.016124119982123375,0.0018556242575868964,0.0034389242064207792,-0.009823138825595379,0.011059673503041267,-0.0040638200007379055,-0.0015785341383889318,-0.001208073808811605,0.003828749991953373,-0.004257441032677889,-0.0011428623693063855,-0.001076644053682685,-0.0003779782564379275,3.0206042538338806e-06,-0.000260510336374864,-0.00014149851631373167,0.0038328110240399837,-0.0004281202855054289,-0.0014291484840214252,-0.0009217541082762182,0.0006938663427717984,0.00043359093251638114,-0.0035708239302039146,-0.0004966544220224023,0.0008070718613453209,0.0005420306115411222,-0.0005315592279657722,-0.0007152119651436806,-0.0010380810126662254,-0.0009492752142250538,-8.538240945199504e-05,0.0002774928288999945,0.0021076432894915342,-0.0001976146304514259,-0.0016398524167016149,0.11626167595386505,0.00025017315056174994,-0.0014669395750388503,-0.0003969745012000203,0.01896214485168457,-0.00018882131553255022,0.011170395649969578,-0.0013304082676768303,-0.0007363561890088022,-0.00030310085276141763,-0.00014579448907170445,-0.0002225178322987631,-0.0016502231592312455,0.00029299859306775033,-0.0014352232683449984,0.030841777101159096,-0.007431894540786743,-0.0002820908557623625,0.006017110776156187,-0.011006947606801987,-0.0012651296565309167,0.001490030437707901,-0.00018016225658357143,0.0029450219590216875,-0.004210856277495623,0.0029608632903546095,0.002045620232820511,0.0013410139363259077,-0.0012189144035801291,0.34349939227104187,0.0005627049831673503,-0.0001257913827430457,-0.005152345169335604,0.016241183504462242,0.017089638859033585,-0.004173669498413801,0.039775215089321136,0.015227536670863628,-0.001023011514917016,0.0008081123232841492,-0.004935599863529205,-0.002123551908880472,-0.014273260720074177,0.0013749118661507964,0.0014838549541309476,0.1302708089351654,-0.00033538774005137384,0.0012914760736748576,0.0003724069392774254,0.0195147804915905,0.00022308839834295213,0.12424899637699127,-0.0004041232750751078,-0.007652163505554199,0.0013012762647122145,-0.0011244030902162194,-0.007449581753462553,0.19224178791046143,-0.0032753082923591137,-0.000501957256346941,-0.001007740618661046,3.1212908652378246e-05,-0.0008586570620536804,0.012360110878944397,-0.00040429108776152134,-0.004328995477408171,0.31855517625808716,0.002329523442313075,0.002118081087246537,0.0001410957775078714,0.27793484926223755,0.005738543812185526,0.0058897752314805984,-0.000968372100032866,0.009125681594014168,0.020675498992204666,-0.03700448200106621,0.014263628050684929,-0.04828385263681412,0.05834108963608742,0.0006516450084745884,0.2636081576347351,0.0004918886115774512,-0.0026110773906111717,0.08374276012182236,0.020676370710134506,-0.0037437702994793653,0.01085111778229475,-0.0010960765648633242,0.0004738992138300091,0.04817944020032883,-0.47990989685058594,0.00018412260396871716,0.011861777864396572,0.06088537722826004,0.0008459034143015742,0.005328110884875059,-0.011493667028844357,-0.11350765824317932,0.006330112461000681,0.0003166264505125582,-0.0011600126745179296,-0.022669030353426933,0.004070901777595282,0.007316238712519407,-0.008345123380422592,-0.27817434072494507,0.003635196480900049],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"Attention Patch"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"Output Patch"}},"legend":{"tracegroupgap":0},"title":{"text":"Scatter plot of output patching vs attention patching"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('d6f23061-00f8-4a84-9d17-cede03fea051');
+var gd = document.getElementById('a4bdc2e4-754f-45ec-9243-67a82b72f8a2');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -1853,27 +1853,27 @@ <h3>Visualizing Attention Patterns<a class="headerlink" href="#Visualizing-Atten
 </pre></div>
 </div>
 <div class="output_area rendered_html docutils container">
-<div style='max-width: 700px;'><h2>Top Early Heads</h2><br/><div id="circuits-vis-cec38c2d-008b" style="margin: 15px 0;"/>
+<div style='max-width: 700px;'><h2>Top Early Heads</h2><br/><div id="circuits-vis-42e293d6-6a8c" style="margin: 15px 0;"/>
     <script crossorigin type="module">
     import { render, AttentionHeads } from "https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js";
     render(
-      "circuits-vis-cec38c2d-008b",
+      "circuits-vis-42e293d6-6a8c",
       AttentionHeads,
       {"attention": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9964697360992432, 0.0035302219912409782, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9966080188751221, 0.0017803364899009466, 0.0016116512706503272, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9577597975730896, 0.002041348023340106, 0.00576710794121027, 0.03443171828985214, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9969283938407898, 0.0006081241299398243, 0.0016115037724375725, 0.0005201429594308138, 0.0003317868977319449, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9679325819015503, 0.0013422606280073524, 0.011834591627120972, 0.004275884013622999, 0.005249680485576391, 0.009365071542561054, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9869164228439331, 0.0013478520559147, 0.008249778300523758, 0.00044454942690208554, 0.0012886020122095942, 0.0008496184600517154, 0.0009031315566971898, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9841058850288391, 0.005145575385540724, 0.0032505656126886606, 0.00023865938419476151, 0.0014756217133253813, 0.000972211651969701, 0.00134052406065166, 0.003471038304269314, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9905135631561279, 0.0026424615643918514, 0.0020855581387877464, 4.1533587591402465e-07, 1.741811320243869e-05, 0.0003096052969340235, 0.00018154106510337442, 0.0010755121475085616, 0.0031737794633954763, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9528716206550598, 0.00732765905559063, 0.008451228961348534, 0.00028046435909345746, 0.002839941531419754, 0.002773231128230691, 0.001109760720282793, 0.0011476994259282947, 0.0057571399956941605, 0.017441321164369583, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04626865312457085, 0.00023600729764439166, 0.00012578870519064367, 0.5745652914047241, 0.3546755015850067, 0.0192879606038332, 9.036171832121909e-05, 0.00044691169750876725, 0.004133324604481459, 3.63319622920244e-06, 0.00016656972002238035, 0.0, 0.0, 0.0, 0.0], [0.887678861618042, 0.005508160684257746, 0.008590048179030418, 0.022789321839809418, 0.034917183220386505, 0.001486042165197432, 7.143727270886302e-05, 0.0006066859932616353, 0.010601978749036789, 0.0001475940807722509, 0.004989828448742628, 0.022612828761339188, 0.0, 0.0, 0.0], [0.9325922131538391, 0.009143135510385036, 0.0023391752038151026, 0.0008593837847001851, 0.0012484431499615312, 0.00037233170587569475, 0.0001396290463162586, 5.667153527610935e-05, 0.029758313670754433, 0.002666445914655924, 0.00340129598043859, 0.0010437102755531669, 0.016379283741116524, 0.0, 0.0], [0.9817627668380737, 0.001917588640935719, 0.00040775517118163407, 1.3564374512498034e-07, 1.264090315089561e-05, 2.9349219403229654e-05, 3.720399035955779e-05, 1.9605669876909815e-05, 0.00034280188265256584, 0.0020146335009485483, 0.007377031724900007, 4.751105734612793e-06, 0.0004324739275034517, 0.005641316995024681, 0.0], [0.908775269985199, 0.004100896418094635, 0.0067780399695038795, 1.580042953719385e-05, 0.00011193934915354475, 8.643363980809227e-05, 3.187262109349831e-06, 0.001585155026987195, 0.003799399361014366, 0.0067048450000584126, 0.0139771094545722, 0.0001160897736554034, 0.004422870930284262, 0.021179812029004097, 0.028343213722109795]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9990116357803345, 0.0009883929742500186, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9985806941986084, 0.0004404807696118951, 0.0009789254982024431, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9970335960388184, 0.0004637633392121643, 0.0009368518949486315, 0.0015657964395359159, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9892035126686096, 8.868379518389702e-05, 0.000574222591239959, 0.009837915189564228, 0.0002956092939712107, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9954046010971069, 0.00010050175478681922, 0.0013415871653705835, 3.886737977154553e-05, 0.0001206161396112293, 0.00299392850138247, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9989855885505676, 0.00010351995297241956, 0.0004172757326159626, 7.227508376672631e-06, 6.046874841558747e-05, 0.0002549678029026836, 0.0001708982017589733, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9980852603912354, 0.00016822919133119285, 0.00010910424316534773, 4.1012435758602805e-06, 5.103566945763305e-05, 0.0001056204637279734, 4.150096719968133e-05, 0.0014351928839460015, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9972643852233887, 7.076081237755716e-05, 6.327207665890455e-05, 2.615450966914068e-08, 5.106949174660258e-06, 3.7383753806352615e-05, 3.455430851317942e-05, 0.000461774121504277, 0.0020626918412745, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9859043955802917, 0.0015692580491304398, 0.001985961804166436, 5.8834184528677724e-06, 0.0003162050561513752, 0.00014976764214225113, 2.4109986043185927e-05, 0.00012667303963098675, 0.0012655562022700906, 0.00865224003791809, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3212882876396179, 0.0026113800704479218, 0.0017525857547298074, 0.48284193873405457, 0.1752953678369522, 0.012053580023348331, 5.7314944569952786e-05, 0.00047417485620826483, 0.0011855909833684564, 4.6230838051997125e-05, 0.0023935972712934017, 0.0, 0.0, 0.0, 0.0], [0.9928818941116333, 0.0011452743783593178, 0.0015605579828843474, 5.570496796281077e-05, 0.0020913302432745695, 0.0002502372954040766, 9.751672678248724e-07, 1.4541505152010359e-05, 0.0006855707615613937, 3.3003976568579674e-05, 0.0003418456471990794, 0.0009391428320668638, 0.0, 0.0, 0.0], [0.9924144148826599, 0.0006524075288325548, 0.00016266616876237094, 7.948347956698854e-06, 0.00015110620006453246, 6.227314588613808e-05, 7.466151714652369e-07, 4.000507090040628e-07, 0.00244068237952888, 3.030487641808577e-05, 4.814808562514372e-05, 9.094910637941211e-05, 0.003938092850148678, 0.0, 0.0], [0.9971888661384583, 0.00062146270647645, 3.946273500332609e-05, 1.9362538239420246e-08, 1.800630002435355e-06, 1.9080694983131252e-05, 9.158955435850658e-06, 7.050003887343337e-07, 6.766093429178e-05, 0.0008072670316323638, 0.00013612494512926787, 2.204007842010469e-06, 2.116413634212222e-05, 0.0010850975522771478, 0.0], [0.9784426689147949, 0.0038412772119045258, 0.001794080133549869, 9.544757631374523e-06, 0.0001613280619494617, 0.00021571738761849701, 3.438441638081713e-07, 0.0001911988656502217, 0.0018583982018753886, 0.002262116875499487, 0.0028807998169213533, 0.00015137717127799988, 3.800494960159995e-05, 0.004419004078954458, 0.0037341148126870394]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9793283343315125, 0.020671749487519264, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.906032145023346, 0.0005559856654144824, 0.09341184794902802, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9916713237762451, 0.0009248568094335496, 0.0019154187757521868, 0.005488428752869368, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9500631093978882, 0.0015093885594978929, 0.00022879372409079224, 0.0011742664501070976, 0.04702451452612877, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9742831587791443, 0.001334858825430274, 0.00016252745990641415, 0.001139267929829657, 1.2687537491729017e-06, 0.023079030215740204, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9734269380569458, 0.0036638062447309494, 0.0007961394148878753, 0.0020336511079221964, 1.7259832020499744e-05, 0.013762583956122398, 0.006299513392150402, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9740443229675293, 0.0018358685774728656, 0.00031735500670038164, 0.0013147101271897554, 4.377634832053445e-05, 0.00045178644359111786, 0.00035931551246903837, 0.021632865071296692, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8537226319313049, 0.0009385455632582307, 1.8928734789369628e-05, 0.0007116434862837195, 4.0574541344540194e-05, 2.706850500544533e-05, 0.0005688024684786797, 0.01303892396390438, 0.1309327930212021, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9718613028526306, 0.0030212444253265858, 0.0018890955252572894, 0.0058778319507837296, 0.0006584360380657017, 0.002494583837687969, 0.001156346406787634, 0.008928273804485798, 0.0013289419002830982, 0.0027839455287903547, 0.0, 0.0, 0.0, 0.0, 0.0], [0.21994593739509583, 0.0028093564324080944, 0.6635686159133911, 0.007326209452003241, 0.001153560820966959, 0.0018521154997870326, 0.0013647314626723528, 0.008370185270905495, 0.0001247532054549083, 0.002670685760676861, 0.09081388264894485, 0.0, 0.0, 0.0, 0.0], [0.9398402571678162, 0.0037398827262222767, 0.0006579139735549688, 0.0027551481034606695, 1.3964868230686989e-05, 0.002081213751807809, 0.007874594070017338, 0.008091994561254978, 0.002785064745694399, 0.003697305917739868, 0.00012857612455263734, 0.028334153816103935, 0.0, 0.0, 0.0], [0.9256278872489929, 0.003680802881717682, 0.0006159098702482879, 0.004310118965804577, 7.914558227639645e-05, 0.0015225791139528155, 0.0014093437930569053, 0.041066594421863556, 0.0011927677551284432, 0.002704742830246687, 8.953329233918339e-05, 0.00031252068583853543, 0.017388159409165382, 0.0, 0.0], [0.7785975933074951, 0.005006679333746433, 8.588898344896734e-05, 0.0016439110040664673, 3.517184086376801e-05, 0.0007985705742612481, 0.004088219255208969, 0.03206512704491615, 0.02768608182668686, 0.003920219838619232, 2.586934351711534e-05, 0.0010157276410609484, 0.009296733886003494, 0.13573415577411652, 0.0], [0.954503059387207, 0.006336927879601717, 0.00032585320877842605, 0.0035448195412755013, 2.0907922589685768e-05, 0.007449527736753225, 0.009912977926433086, 0.005456207785755396, 0.0005529882037080824, 0.0011178080458194017, 6.70458102831617e-05, 0.003978283144533634, 0.0024476239923387766, 6.584440416190773e-05, 0.00422010850161314]]], "attentionHeadNames": ["L5H5", "L6H9", "L3H0"], "tokens": ["<|endoftext|>", "When", " John", " and", " Mary", " went", " to", " the", " shops", ",", " John", " gave", " the", " bag", " to"]}
     )
-    </script></div><div style='max-width: 700px;'><h2>Top Middle Heads</h2><br/><div id="circuits-vis-4796b044-6a22" style="margin: 15px 0;"/>
+    </script></div><div style='max-width: 700px;'><h2>Top Middle Heads</h2><br/><div id="circuits-vis-17a2a2ac-56a8" style="margin: 15px 0;"/>
     <script crossorigin type="module">
     import { render, AttentionHeads } from "https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js";
     render(
-      "circuits-vis-4796b044-6a22",
+      "circuits-vis-17a2a2ac-56a8",
       AttentionHeads,
       {"attention": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9883801937103271, 0.011619755066931248, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9756757020950317, 0.006998097989708185, 0.017326191067695618, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.06910058110952377, 0.005814341828227043, 0.9046230316162109, 0.020462043583393097, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8063386082649231, 0.003319838549941778, 0.0537770576775074, 0.11000825464725494, 0.02655618265271187, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9564918875694275, 0.004251283593475819, 0.011477907188236713, 0.011219605803489685, 0.005006618797779083, 0.011552776210010052, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9077332615852356, 0.0036374430637806654, 0.040214236825704575, 0.012946903705596924, 0.019611867144703865, 0.011057589203119278, 0.004798637703061104, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9552405476570129, 0.009812522679567337, 0.01168749388307333, 0.0090243611484766, 0.0034366166219115257, 0.004399922210723162, 0.0033402289263904095, 0.003058257745578885, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9294586777687073, 0.009998402558267117, 0.004449557047337294, 0.006221562158316374, 0.0014876301866024733, 0.018290331587195396, 0.00712206494063139, 0.0036232024431228638, 0.0193486325442791, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6007111668586731, 0.014475381001830101, 0.09337030351161957, 0.01660209521651268, 0.05098922923207283, 0.10098620504140854, 0.051671840250492096, 0.02743108756840229, 0.02603602036833763, 0.017726697027683258, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6264241933822632, 0.012645086273550987, 0.016070131212472916, 0.024194441735744476, 0.029195599257946014, 0.11857090145349503, 0.06370382010936737, 0.023378603160381317, 0.001658876077271998, 0.06528978794813156, 0.018868643790483475, 0.0, 0.0, 0.0, 0.0], [0.2991042137145996, 0.0021427334286272526, 0.024437544867396355, 0.005155442748218775, 0.010526876896619797, 0.003321276046335697, 0.009254480712115765, 0.005843525752425194, 0.0021181809715926647, 0.017681695520877838, 0.6048213839530945, 0.015592672862112522, 0.0, 0.0, 0.0], [0.8641641139984131, 0.004937537480145693, 0.019023817032575607, 0.001670254161581397, 0.00609729764983058, 0.0005472623743116856, 0.003564003622159362, 0.00030002548010088503, 0.007773987948894501, 0.007786957547068596, 0.06421297788619995, 0.005789309274405241, 0.014132493175566196, 0.0, 0.0], [0.9178580045700073, 0.005138528533279896, 0.003403181442990899, 0.0021888096816837788, 0.002614964498206973, 0.0028868599329143763, 0.0028416835702955723, 0.0013745100004598498, 0.006431094836443663, 0.012306848540902138, 0.0090817641466856, 0.01310056634247303, 0.008110282011330128, 0.012662900611758232, 0.0], [0.019487837329506874, 5.700236215488985e-05, 0.026276404038071632, 0.0009103945340029895, 0.0056328438222408295, 7.981786620803177e-05, 5.7736750022741035e-05, 0.000307683803839609, 0.00014904775889590383, 0.0002852973702829331, 0.9430450201034546, 0.0007767666247673333, 0.0018659740453585982, 0.0006072799442335963, 0.0004609254829119891]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9562382698059082, 0.043761663138866425, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9643059372901917, 0.00742809334769845, 0.028266001492738724, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6456211805343628, 0.029121818020939827, 0.20926059782505035, 0.11599643528461456, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7634043097496033, 0.01714019477367401, 0.026512376964092255, 0.08158667385578156, 0.11135648936033249, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8267454504966736, 0.029910113662481308, 0.015236063860356808, 0.034572526812553406, 0.027030833065509796, 0.06650502979755402, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7882238626480103, 0.02833343856036663, 0.04124196991324425, 0.024983419105410576, 0.05788382515311241, 0.020290205255150795, 0.03904324769973755, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7217290997505188, 0.027771806344389915, 0.02081979066133499, 0.024504197761416435, 0.01813969761133194, 0.014079142361879349, 0.04442400485277176, 0.12853237986564636, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7728042602539062, 0.04133640229701996, 0.006504536606371403, 0.007949033752083778, 0.006013217847794294, 0.01562037505209446, 0.03980163484811783, 0.05545476824045181, 0.054515715688467026, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.47073426842689514, 0.03082239255309105, 0.07406120002269745, 0.055252522230148315, 0.14865727722644806, 0.062336407601833344, 0.034053124487400055, 0.026126855984330177, 0.042438995093107224, 0.05551688000559807, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6962025165557861, 0.0052972170524299145, 0.004229621961712837, 0.05190793052315712, 0.007751153316348791, 0.03517890349030495, 0.03057355433702469, 0.021498825401067734, 0.01566012017428875, 0.051384858787059784, 0.08031534403562546, 0.0, 0.0, 0.0, 0.0], [0.37817656993865967, 0.006224599666893482, 0.023588571697473526, 0.044115372002124786, 0.059451501816511154, 0.00953902117908001, 0.012997719459235668, 0.014349035918712616, 0.0203937329351902, 0.01785479485988617, 0.33459627628326416, 0.07871285080909729, 0.0, 0.0, 0.0], [0.4760020971298218, 0.011026055552065372, 0.018324486911296844, 0.04565680772066116, 0.026791490614414215, 0.011260150000452995, 0.018724240362644196, 0.018448904156684875, 0.04068983346223831, 0.023096060380339622, 0.12447667866945267, 0.08994489908218384, 0.0955582857131958, 0.0, 0.0], [0.7641531229019165, 0.00824385043233633, 0.0032178766559809446, 0.004948926158249378, 0.004036621656268835, 0.00750678451731801, 0.011691190302371979, 0.013843986205756664, 0.01143654901534319, 0.0222361721098423, 0.013085336424410343, 0.03330926224589348, 0.036730363965034485, 0.06555990874767303, 0.0], [0.1303366869688034, 0.0022374859545379877, 0.0373632088303566, 0.06385007500648499, 0.11956273019313812, 0.011286774650216103, 0.006565204821527004, 0.00851503573358059, 0.005633368622511625, 0.007633764296770096, 0.5373668074607849, 0.0301155224442482, 0.020915960893034935, 0.005129658151417971, 0.013487689197063446]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.972596287727356, 0.027403652667999268, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9560405015945435, 0.037302762269973755, 0.006656836252659559, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5009381771087646, 0.0898313894867897, 0.34842997789382935, 0.06080043688416481, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7674180269241333, 0.17525732517242432, 0.021164551377296448, 0.02390032634139061, 0.012259737588465214, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7222559452056885, 0.18388347327709198, 0.023735947906970978, 0.007100591901689768, 0.007225400768220425, 0.05579861253499985, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6940793395042419, 0.13069462776184082, 0.028726812452077866, 0.009216434322297573, 0.011527394875884056, 0.08933128416538239, 0.036424122750759125, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6924421191215515, 0.12459339946508408, 0.01531891617923975, 0.007925059646368027, 0.006713555194437504, 0.0796525627374649, 0.04910353198647499, 0.02425084263086319, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6969695091247559, 0.10134164988994598, 0.011634294874966145, 0.007826745510101318, 0.0032780379988253117, 0.042647555470466614, 0.05842007324099541, 0.042913682758808136, 0.03496843948960304, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6869087815284729, 0.023936893790960312, 0.10650163143873215, 0.02047020196914673, 0.02552187815308571, 0.04722552374005318, 0.022190820425748825, 0.007246002089232206, 0.019754212349653244, 0.04024401307106018, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5810369849205017, 0.011352411471307278, 0.0033976146951317787, 0.0365644209086895, 0.020549381151795387, 0.051692016422748566, 0.024651870131492615, 0.015468521043658257, 0.049509111791849136, 0.15767459571361542, 0.04810298979282379, 0.0, 0.0, 0.0, 0.0], [0.43488532304763794, 0.006897220853716135, 0.012041947804391384, 0.003814741736277938, 0.010545267723500729, 0.0076634688302874565, 0.007842864841222763, 0.006417531985789537, 0.044214341789484024, 0.10391034185886383, 0.3332052230834961, 0.028561806306242943, 0.0, 0.0, 0.0], [0.4747830927371979, 0.024993136525154114, 0.009554888121783733, 0.0037503365892916918, 0.008160687983036041, 0.02339756116271019, 0.010678932070732117, 0.0014935131184756756, 0.030863573774695396, 0.19498929381370544, 0.15084564685821533, 0.054150503128767014, 0.012338940985500813, 0.0, 0.0], [0.7144870758056641, 0.009456266649067402, 0.0035061524249613285, 0.0029858576599508524, 0.001229096669703722, 0.01199608389288187, 0.008696629665791988, 0.007586767431348562, 0.009631272405385971, 0.06210443004965782, 0.014414231292903423, 0.07928328961133957, 0.04383841156959534, 0.030784491449594498, 0.0], [0.1013382077217102, 0.0038837045431137085, 0.01792380027472973, 0.0038827748503535986, 0.009177407249808311, 0.01316938828676939, 0.0034797503612935543, 0.0019711176864802837, 0.004929723683744669, 0.1296602189540863, 0.6630275249481201, 0.0330243818461895, 0.0089839156717062, 0.0017172213410958648, 0.003830986563116312]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.97651207447052, 0.023487908765673637, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9743125438690186, 0.009410635568201542, 0.016276877373456955, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5441411137580872, 0.019679147750139236, 0.33595189452171326, 0.10022785514593124, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8594159483909607, 0.021821973845362663, 0.025710705667734146, 0.03842515870928764, 0.05462610721588135, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7619735598564148, 0.07210072875022888, 0.037441689521074295, 0.04506158083677292, 0.031744565814733505, 0.05167779326438904, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8025401830673218, 0.026160430163145065, 0.033131979405879974, 0.02643769420683384, 0.03139319270849228, 0.04932845011353493, 0.031008044257760048, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7639790773391724, 0.021559985354542732, 0.016495496034622192, 0.025408243760466576, 0.027263421565294266, 0.06810054928064346, 0.047422830015420914, 0.029770376160740852, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7543362379074097, 0.01597309671342373, 0.008247380144894123, 0.00967292208224535, 0.013097813352942467, 0.03411843255162239, 0.03947027027606964, 0.032298993319272995, 0.09278487414121628, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5856387615203857, 0.0213167741894722, 0.026237597689032555, 0.020694918930530548, 0.02824069745838642, 0.11104536801576614, 0.047170888632535934, 0.022986045107245445, 0.04925474151968956, 0.08741410821676254, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6466761231422424, 0.015195527113974094, 0.00594041682779789, 0.02527434751391411, 0.010863534174859524, 0.08686516433954239, 0.032730914652347565, 0.021324681118130684, 0.04200880602002144, 0.09539096057415009, 0.017729468643665314, 0.0, 0.0, 0.0, 0.0], [0.42771559953689575, 0.014421668834984303, 0.024129344150424004, 0.026094987988471985, 0.01488838903605938, 0.010527816601097584, 0.012212568894028664, 0.015342887490987778, 0.017334001138806343, 0.04439026489853859, 0.28982794284820557, 0.10311443358659744, 0.0, 0.0, 0.0], [0.4405166506767273, 0.018903780728578568, 0.010929570533335209, 0.015466002747416496, 0.013489533215761185, 0.020115477964282036, 0.019655633717775345, 0.005925851874053478, 0.022772811353206635, 0.06206439808011055, 0.16126078367233276, 0.17319339513778687, 0.03570614382624626, 0.0, 0.0], [0.7304301261901855, 0.008421944454312325, 0.0022352896630764008, 0.004027598071843386, 0.0036414540372788906, 0.009633221663534641, 0.0213489830493927, 0.021966980770230293, 0.02610298991203308, 0.05093422904610634, 0.0083627263084054, 0.023242846131324768, 0.038515783846378326, 0.051135752350091934, 0.0], [0.16577893495559692, 0.004480816423892975, 0.0077864062041044235, 0.007284588646143675, 0.01337976660579443, 0.019251490011811256, 0.010158197022974491, 0.01172470673918724, 0.011870641261339188, 0.034697193652391434, 0.1448296457529068, 0.4143996834754944, 0.08938297629356384, 0.019558662548661232, 0.04541636258363724]]], "attentionHeadNames": ["L8H6", "L8H10", "L7H9", "L7H3"], "tokens": ["<|endoftext|>", "When", " John", " and", " Mary", " went", " to", " the", " shops", ",", " John", " gave", " the", " bag", " to"]}
     )
-    </script></div><div style='max-width: 700px;'><h2>Top Late Heads</h2><br/><div id="circuits-vis-3f1d1fde-8de5" style="margin: 15px 0;"/>
+    </script></div><div style='max-width: 700px;'><h2>Top Late Heads</h2><br/><div id="circuits-vis-abec0760-a70a" style="margin: 15px 0;"/>
     <script crossorigin type="module">
     import { render, AttentionHeads } from "https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js";
     render(
-      "circuits-vis-3f1d1fde-8de5",
+      "circuits-vis-abec0760-a70a",
       AttentionHeads,
       {"attention": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9628916382789612, 0.03710832819342613, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9778335094451904, 0.0034864130429923534, 0.018680110573768616, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8737697601318359, 0.005964328069239855, 0.08126737922430038, 0.03899851813912392, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8745138645172119, 0.017276111990213394, 0.018874678760766983, 0.054268334060907364, 0.035066962242126465, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9704693555831909, 0.0035177830141037703, 0.0003239834331907332, 0.004373463336378336, 0.0003953034174628556, 0.02092011272907257, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7694658041000366, 0.006685878615826368, 0.011966485530138016, 0.040966276079416275, 0.03344443440437317, 0.04401002824306488, 0.0934610366821289, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8177722692489624, 0.009112595580518246, 0.0042336746118962765, 0.03900325670838356, 0.012653318233788013, 0.007337852846831083, 0.07410021126270294, 0.0357869453728199, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8309933543205261, 0.018796611577272415, 0.00330327614210546, 0.01983213610947132, 0.004009497817605734, 0.012078307569026947, 0.03246007487177849, 0.010322270914912224, 0.06820447742938995, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.31403300166130066, 0.01522789802402258, 0.19789263606071472, 0.059379592537879944, 0.3236042559146881, 0.009037697687745094, 0.019917670637369156, 0.012475269846618176, 0.013939663767814636, 0.03449229896068573, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7319568991661072, 0.03087025322020054, 0.005064091179519892, 0.08703826367855072, 0.004982746671885252, 0.044725559651851654, 0.02350614219903946, 0.0063977413810789585, 0.0034696452785283327, 0.048653144389390945, 0.013335512951016426, 0.0, 0.0, 0.0, 0.0], [0.13710583746433258, 0.012017901986837387, 0.14592327177524567, 0.015683427453041077, 0.44125232100486755, 0.004342416767030954, 0.011117888614535332, 0.007466601673513651, 0.0012341630645096302, 0.01946023479104042, 0.16842849552631378, 0.035967420786619186, 0.0, 0.0, 0.0], [0.4052484929561615, 0.009720955975353718, 0.03158125653862953, 0.09247038513422012, 0.048808369785547256, 0.008174203336238861, 0.031708307564258575, 0.02787904627621174, 0.14128026366233826, 0.0302340779453516, 0.03392859548330307, 0.03171432018280029, 0.10725171118974686, 0.0, 0.0], [0.808469295501709, 0.0056702690199017525, 0.006755992770195007, 0.008696584962308407, 0.010473440401256084, 0.003464736510068178, 0.006760997697710991, 0.0025889205280691385, 0.015097705647349358, 0.01603602059185505, 0.011288060806691647, 0.017277495935559273, 0.00779919745400548, 0.07962135970592499, 0.0], [0.02939111366868019, 0.0016060526249930263, 0.06005561351776123, 0.008175727911293507, 0.8099164366722107, 0.00039796155760996044, 0.0028282543644309044, 0.0033632616978138685, 0.0001569116284372285, 0.0036459483671933413, 0.05494074895977974, 0.001666039228439331, 0.015368940308690071, 5.016689101466909e-05, 0.008436810225248337]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9124024510383606, 0.08759760111570358, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.97407466173172, 0.01434413529932499, 0.011581187136471272, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8240247964859009, 0.03700513392686844, 0.06643965095281601, 0.07253041118383408, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5370795726776123, 0.15483324229717255, 0.0903128981590271, 0.12131187319755554, 0.0964624434709549, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9388002157211304, 0.028733234852552414, 0.0003431019140407443, 0.010481251403689384, 0.000839930260553956, 0.02080223336815834, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6318807005882263, 0.059302832931280136, 0.03517911583185196, 0.0317409411072731, 0.1174689382314682, 0.06869616359472275, 0.05573126673698425, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7625216841697693, 0.04435169696807861, 0.010552966967225075, 0.034276288002729416, 0.014203979633748531, 0.012703591957688332, 0.07259687036275864, 0.048792947083711624, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5725991725921631, 0.12681783735752106, 0.03727060556411743, 0.0431414395570755, 0.04129580408334732, 0.046447426080703735, 0.029164383187890053, 0.03751189261674881, 0.06575141847133636, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12316936254501343, 0.026347072795033455, 0.27960509061813354, 0.041989635676145554, 0.44105827808380127, 0.006138973869383335, 0.007793003693223, 0.007175281643867493, 0.011286810971796513, 0.055436521768569946, 0.0, 0.0, 0.0, 0.0, 0.0], [0.44855940341949463, 0.1432485729455948, 0.010774216614663601, 0.07855041325092316, 0.01404787041246891, 0.014864671975374222, 0.008633698336780071, 0.0030146560166031122, 0.012792127206921577, 0.16408587992191315, 0.10142850130796432, 0.0, 0.0, 0.0, 0.0], [0.10228201746940613, 0.0238832775503397, 0.055250268429517746, 0.0520680733025074, 0.5759841799736023, 0.0011948152678087354, 0.0013662497512996197, 0.0025677781086415052, 0.0018699930515140295, 0.043160032480955124, 0.10887196660041809, 0.03150142729282379, 0.0, 0.0, 0.0], [0.39313194155693054, 0.04406267777085304, 0.04009911045432091, 0.07407406717538834, 0.04061643034219742, 0.007346798665821552, 0.009507199749350548, 0.013865168206393719, 0.0640539675951004, 0.04894229769706726, 0.0627753809094429, 0.0999651551246643, 0.10155977308750153, 0.0, 0.0], [0.7573198676109314, 0.015095721930265427, 0.007350177504122257, 0.013156400062143803, 0.005726401228457689, 0.008556416258215904, 0.008282607421278954, 0.010307075455784798, 0.002904881490394473, 0.017329057678580284, 0.015188980847597122, 0.0796574279665947, 0.030244464054703712, 0.02888045459985733, 0.0], [0.08579524606466293, 0.023773644119501114, 0.028759371489286423, 0.050303805619478226, 0.674644947052002, 0.0010763936443254352, 0.0011514866491779685, 0.002721975091844797, 0.0005617713322862983, 0.039091579616069794, 0.04861969128251076, 0.015520088374614716, 0.0040845321491360664, 0.00043291584006510675, 0.023462524637579918]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9979304075241089, 0.002069620881229639, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9971864819526672, 0.0010516609763726592, 0.0017618348356336355, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9594857096672058, 0.001310725579969585, 0.03694308176636696, 0.002260456094518304, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9891613125801086, 0.001052629784680903, 0.0048541901633143425, 0.001104357186704874, 0.003827564185485244, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9963598847389221, 0.0007981774397194386, 0.0007764195324853063, 0.00019262291607446969, 0.0002416159404674545, 0.0016313291853293777, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.981286346912384, 0.001863775309175253, 0.006344134453684092, 0.0002960922720376402, 0.004604941233992577, 0.0013966941041871905, 0.00420788861811161, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9950011372566223, 0.0013230532640591264, 0.0007920759962871671, 0.00024644460063427687, 0.0003405151073820889, 0.00016826140927150846, 0.0002837753272615373, 0.0018447580514475703, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9920291900634766, 0.0013539871433749795, 0.0007168339216150343, 9.214139572577551e-05, 0.00013419234892353415, 0.00019706363673321903, 0.00035278062568977475, 0.0002426079590804875, 0.004881155211478472, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5483196377754211, 0.0075838202610611916, 0.28265973925590515, 0.0029685653280466795, 0.14815661311149597, 0.0008172960951924324, 0.0006479909643530846, 0.0017078499076887965, 0.004129356704652309, 0.003009046893566847, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8642559051513672, 0.0036493062507361174, 0.004925842396914959, 0.06957440823316574, 0.038683053106069565, 0.009609837085008621, 0.0008548588375560939, 0.0005643004551529884, 0.003707138355821371, 0.0013952680164948106, 0.002780066104605794, 0.0, 0.0, 0.0, 0.0], [0.30252957344055176, 0.006584456656128168, 0.1401754468679428, 0.030065739527344704, 0.4806697964668274, 0.0005923578282818198, 0.0005309724947437644, 0.0024134721606969833, 0.008505754172801971, 0.0013766245683655143, 0.023764921352267265, 0.0027908117044717073, 0.0, 0.0, 0.0], [0.7773900032043457, 0.003722917754203081, 0.022378109395503998, 0.015139483846724033, 0.017408354207873344, 0.002416662173345685, 0.000709561922121793, 0.0007399603491649032, 0.13850137591362, 0.0023606533650308847, 0.005776867736130953, 0.0028014343697577715, 0.010654616169631481, 0.0, 0.0], [0.9735962152481079, 0.0012812522472813725, 0.002617918187752366, 9.891873924061656e-05, 0.0005098494002595544, 0.00012000725837424397, 0.00045229491661302745, 0.00010779645526781678, 0.002954537281766534, 0.001644388772547245, 0.0012141242623329163, 0.00018637391622178257, 0.0008510759216733277, 0.01436527632176876, 0.0], [0.0990300253033638, 0.000971083587501198, 0.06413238495588303, 0.005246965680271387, 0.8110400438308716, 8.250321843661368e-05, 6.596777529921383e-05, 0.0011991349747404456, 0.0008852760074660182, 0.00024891181965358555, 0.010336406528949738, 7.952339365147054e-05, 0.0049433172680437565, 0.00028918671887367964, 0.0014492359478026628]]], "attentionHeadNames": ["L10H7", "L11H10", "L9H9"], "tokens": ["<|endoftext|>", "When", " John", " and", " Mary", " went", " to", " the", " shops", ",", " John", " gave", " the", " bag", " to"]}
     )
@@ -1955,11 +1955,11 @@ <h3>Early Heads are Induction Heads(?!)<a class="headerlink" href="#Early-Heads-
 </pre></div>
 </div>
 <div class="output_area rendered_html docutils container">
-<div style='max-width: 800px;'><h2>Induction Heads</h2><br/><div id="circuits-vis-2ebcde31-65b5" style="margin: 15px 0;"/>
+<div style='max-width: 800px;'><h2>Induction Heads</h2><br/><div id="circuits-vis-eeb72893-5fb3" style="margin: 15px 0;"/>
     <script crossorigin type="module">
     import { render, AttentionHeads } from "https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js";
     render(
-      "circuits-vis-2ebcde31-65b5",
+      "circuits-vis-eeb72893-5fb3",
       AttentionHeads,
       {"attention": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9995203018188477, 0.0004796621506102383, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9989193677902222, 0.00011719338363036513, 0.0009634991874918342, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.999607503414154, 4.1751700337044895e-05, 9.65828076004982e-06, 0.00034111665445379913, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9989573955535889, 3.091409962507896e-05, 1.212368533742847e-05, 0.0002459374663885683, 0.0007535542827099562, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9994303584098816, 3.2271585951093584e-05, 2.162250348192174e-05, 0.000438225019024685, 4.1958148358389735e-05, 3.5627250326797366e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.998076319694519, 1.6859989045769908e-05, 2.043985659838654e-06, 6.719817611156031e-05, 1.8109081167949626e-07, 0.00016430420510005206, 0.0016730953939259052, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9989305138587952, 0.0001340252347290516, 1.5985064237611368e-05, 0.000491619051899761, 6.565183241491468e-08, 6.484094228653703e-06, 1.1354626622051e-05, 0.00040989238186739385, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9949487447738647, 0.0005971402861177921, 1.4574082342733163e-05, 0.0018354525091126561, 9.743422424435266e-07, 7.463527435902506e-05, 5.5863079069240484e-06, 0.00020289771782699972, 0.002319993684068322, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9855782985687256, 6.499102164525539e-05, 8.181475277524441e-05, 0.0024699484929442406, 4.049685031759509e-08, 3.498060686979443e-05, 0.001560866367071867, 0.0018861292628571391, 0.00011740217451006174, 0.008205565623939037, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9950633645057678, 0.00021360788377933204, 0.00014613209350500256, 0.00030726491240784526, 2.292262615810614e-05, 0.0002385093830525875, 1.9861789041897282e-05, 0.000960799225140363, 0.0001365413481835276, 0.0016751259099692106, 0.001215968164615333, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9907191395759583, 0.00013381309690885246, 3.014638878084952e-06, 0.0023974282667040825, 2.1563739210250787e-05, 0.0004241704009473324, 1.383451308356598e-05, 5.452497134683654e-05, 2.246501207991969e-05, 0.0017353435978293419, 0.0022565408144146204, 0.002218238776549697, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.99763023853302, 7.048921543173492e-05, 1.2420621260389453e-06, 6.923218461452052e-05, 0.0007476441096514463, 0.000763593299780041, 7.526726676587714e-06, 7.549879228463396e-05, 2.769195930341084e-07, 9.996434528147802e-05, 3.433594974922016e-05, 0.00019309873459860682, 0.00030683254590258, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9413580298423767, 0.0009730295860208571, 0.0004518784990068525, 0.0012102690525352955, 2.08603341889102e-06, 0.0018085278570652008, 1.965607407328207e-05, 0.03966164216399193, 0.0004176551301497966, 0.003035214263945818, 0.002604098990559578, 0.00038386619416996837, 0.007335478439927101, 0.0007384858326986432, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.98450767993927, 0.00010836195724550635, 2.4642587959533557e-05, 0.0004206285811960697, 1.2694325732809375e-06, 0.00011915803042938933, 9.773763304110616e-05, 0.0023661903105676174, 0.00036190610262565315, 0.003964339382946491, 0.0009553474374115467, 0.0006011889781802893, 0.002696628449484706, 0.00010731287329690531, 0.003667621174827218, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9236841797828674, 0.0004220438713673502, 1.1906979125342332e-05, 0.036959629505872726, 0.00012378330575302243, 0.0005373558960855007, 0.0029662158340215683, 0.0006545133655890822, 5.279582546791062e-05, 0.0077215940691530704, 0.0131848668679595, 7.147376891225576e-05, 0.00951047521084547, 0.0007938834605738521, 0.0006112216506153345, 0.0026941129472106695, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9707465171813965, 7.250557973748073e-05, 4.7150632553893956e-07, 0.0007667056052014232, 1.613832296243345e-06, 4.375730350147933e-05, 0.00013073488662485033, 0.0010341654997318983, 0.000541800691280514, 0.006641003768891096, 0.014592697843909264, 0.00018458777049090713, 0.003935785032808781, 3.716452511071111e-06, 2.8841026505688205e-05, 0.0008475544163957238, 0.0004274799139238894, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9573933482170105, 0.00019599858205765486, 2.3228804479913379e-07, 0.003389731515198946, 9.590152103555738e-07, 5.782760854344815e-05, 1.3954500900581479e-05, 9.882948506856337e-05, 0.00014849765284452587, 0.006945967674255371, 0.01562657207250595, 8.081036867224611e-06, 0.011182066984474659, 6.349627255985979e-06, 2.5934892619261518e-05, 0.00012999530008528382, 0.0007364018238149583, 0.004039338324218988, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.990619421005249, 6.466358172474429e-05, 9.901318662741687e-07, 0.00021274133177939802, 1.4796115465287585e-05, 0.00017918922821991146, 6.778116585337557e-06, 7.372434629360214e-05, 0.0002996290859300643, 0.0013087284751236439, 0.0005674119456671178, 0.0011630650842562318, 0.0028649545274674892, 9.635516471462324e-05, 8.433945913566276e-05, 0.00028768807533197105, 4.3501222535269335e-05, 0.00038704575854353607, 0.001724953530356288, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9956457018852234, 2.669214882189408e-05, 3.9460044121142346e-08, 0.001181368250399828, 1.471557879995089e-05, 0.0001445170200895518, 0.0004370114766061306, 3.213761010556482e-05, 8.180093402643251e-08, 9.59495737333782e-05, 0.0007029082044027746, 1.2536466783785727e-05, 0.00018710187578108162, 0.00011147692566737533, 7.006971713963139e-07, 4.411066583998036e-06, 0.0007388183148577809, 4.973216618964216e-06, 9.905266779242083e-05, 0.0005597113631665707, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9956768155097961, 3.464311521383934e-05, 5.037085770709382e-08, 0.00012908628559671342, 3.5288087474327767e-06, 8.07559808890801e-06, 3.11900585074909e-05, 6.715198833262548e-05, 1.5350008197856368e-07, 0.00010576295608188957, 0.0004207944730296731, 4.739598443848081e-05, 0.00017379986820742488, 3.910295163223054e-06, 3.7129705106053734e-06, 6.050187221262604e-05, 4.920005812891759e-05, 4.588661977322772e-05, 0.0002989978529512882, 0.00027882878202944994, 0.0025604318361729383, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9873839616775513, 0.002735754009336233, 1.0963850400003139e-05, 0.0005862272810190916, 6.101510280132061e-06, 6.327711162157357e-05, 2.1911782823735848e-05, 0.00033786968560889363, 6.588418182218447e-05, 0.0013113167369738221, 0.0019715256057679653, 0.00013529015996027738, 0.0018900128779932857, 0.00018075927800964564, 0.00026106255245395005, 0.00037814932875335217, 9.485983173362911e-05, 4.87527868244797e-05, 0.00035683103487826884, 0.0007243757136166096, 0.00010285121243214235, 0.0013322114245966077, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3808109164237976, 0.005228960886597633, 0.6019217371940613, 0.006041430402547121, 3.194508099113591e-05, 4.269973578630015e-05, 8.645826892461628e-05, 0.0006069993833079934, 0.0006299586966633797, 0.0009580003097653389, 0.0006117315497249365, 0.0002489396429155022, 0.0002464656427036971, 6.495700654340908e-05, 0.0006897134589962661, 0.00021990980894770473, 7.002806523814797e-05, 0.00014192514936439693, 2.700363074836787e-05, 5.169477663002908e-05, 4.5723249968432356e-06, 0.00029897037893533707, 0.0009650306892581284, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.563797116279602, 0.0085763493552804, 0.004927160684019327, 0.34193748235702515, 2.1189502149354666e-05, 0.00020753320131916553, 0.0003593186556827277, 0.0014688574010506272, 0.0003070282400585711, 0.02934207208454609, 0.004261575639247894, 8.383052772842348e-06, 0.0024304059334099293, 9.930188389262185e-05, 0.000542300462257117, 9.130241960519925e-05, 0.03218064829707146, 7.990232552401721e-05, 0.0003810952475760132, 0.00010837644367711619, 3.33801144734025e-05, 0.0008215161506086588, 0.0017809885321184993, 0.006236794404685497, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3740951418876648, 0.00012492266250774264, 8.869202247296926e-06, 0.0009476605337113142, 0.6135826706886292, 0.002516972366720438, 0.0002441502292640507, 5.642483665724285e-05, 1.795958496586536e-06, 0.0004439418844413012, 5.296186645864509e-05, 0.0001609584578545764, 0.0007701200665906072, 0.005056836176663637, 1.382048776576994e-05, 5.882297773496248e-05, 0.00018946161435451359, 0.00012655068712774664, 3.178522456437349e-05, 6.542137998621911e-05, 1.7726270016282797e-05, 0.0008768205880187452, 0.00038724290789105, 9.015519140120887e-07, 0.00016804093320388347, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.18932704627513885, 0.0003522285260260105, 1.1454378181952052e-05, 0.0007539974758401513, 0.003070831298828125, 0.7761431336402893, 5.7844576076604426e-05, 0.004086362663656473, 1.6754691387177445e-05, 0.0018927170895040035, 0.00031636765925213695, 0.0005395743064582348, 0.005218177568167448, 0.007962456904351711, 0.004062105901539326, 0.002489584032446146, 1.4082329471420962e-05, 1.4990718227636535e-05, 0.00015581185289192945, 0.0003996991435997188, 5.5086038628360257e-05, 0.0019400393357500434, 0.0003281844255980104, 1.4123389746600878e-07, 8.962309038906824e-06, 0.0007822647457942367, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02010822296142578, 2.635493547131773e-05, 2.7333503567206208e-06, 0.0002452948538120836, 1.4769997505936772e-05, 2.0045878045493737e-05, 0.9765581488609314, 0.0007374248816631734, 2.2332008029479766e-06, 0.00016837129078339785, 0.00137846521101892, 1.0426949756947579e-06, 9.273762407246977e-05, 6.001193469273858e-06, 1.852532113844063e-05, 7.490423013223335e-05, 2.892227712436579e-05, 4.0283404814545065e-05, 1.959171459020581e-05, 2.944982952612918e-05, 8.47621322463965e-06, 0.0002561432193033397, 5.941176277701743e-05, 2.014576523379219e-07, 7.743915921309963e-05, 7.494620604120428e-06, 1.74276083271252e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.22277945280075073, 0.002593980636447668, 2.9011016522417776e-06, 0.0005522919236682355, 1.904411533359962e-06, 0.0006727795698679984, 0.0021306686103343964, 0.6761497855186462, 6.977441080380231e-05, 0.003776289988309145, 0.06293372064828873, 0.0015865935711190104, 0.0017082190606743097, 3.881497832480818e-05, 0.009209989570081234, 0.0013166061835363507, 0.00018038915004581213, 0.001267515355721116, 0.0018657379550859332, 0.0009925103513523936, 3.7589063140330836e-05, 0.006426853593438864, 0.0020985312294214964, 3.3772406027310353e-07, 2.868737283279188e-05, 2.698123182653944e-07, 0.00025180247030220926, 0.001325962133705616, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08230321109294891, 0.0005252842092886567, 4.260616606188705e-06, 0.0006264003459364176, 8.583574384601889e-08, 4.224356871418422e-06, 4.716531293524895e-06, 0.0003961432375945151, 0.8586453795433044, 0.05151975154876709, 0.0026339711621403694, 1.6450698240078054e-05, 0.0006709631998091936, 1.5013217762316344e-06, 9.040276927407831e-05, 7.098149944795296e-05, 7.727313459326979e-06, 0.00020618800772354007, 0.0005295966984704137, 0.0013012847630307078, 5.2623075674773645e-08, 9.972530097002164e-05, 0.00018814747454598546, 2.380612386332359e-06, 6.255433254409581e-05, 3.8025717685741256e-08, 9.557845714880386e-07, 3.836729320028098e-06, 8.390715811401606e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.465234637260437, 0.004093279130756855, 6.743345238646725e-06, 0.004121774807572365, 5.443537247629138e-07, 5.419518129201606e-05, 9.294981850871409e-07, 0.00026017383788712323, 0.00907624140381813, 0.5008078813552856, 0.0023183668963611126, 2.4238510377472267e-05, 0.0030431346967816353, 4.68218931928277e-05, 0.00012087374489055946, 0.0001290314394282177, 0.00011785242531914264, 4.014383375761099e-05, 0.0024619202595204115, 0.0034230302553623915, 1.7263497511521564e-06, 0.0002791083534248173, 0.002281592693179846, 4.889936462859623e-06, 0.00025089550763368607, 2.8178769184705743e-07, 1.050695664162049e-05, 1.070987195816997e-06, 4.786956196767278e-05, 0.0017402973026037216, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3585144281387329, 0.0011027655564248562, 0.00016592990141361952, 0.013357667252421379, 2.546785253798589e-07, 6.185192614793777e-05, 0.0057292357087135315, 0.010611956007778645, 0.0007551729213446379, 0.06400889158248901, 0.4949103593826294, 8.391502342419699e-05, 0.005665580276399851, 2.122150362993125e-06, 0.0002814042672980577, 0.0004684626474045217, 0.0004158185620326549, 0.0012447141343727708, 0.012713258154690266, 0.0010433174902573228, 1.1126050594612025e-05, 0.0011420734226703644, 0.0006368785980157554, 7.042537617962807e-05, 0.002744977129623294, 1.2284903050385765e-07, 1.483806499891216e-05, 0.005110186990350485, 0.004603618290275335, 0.00015696643094997853, 0.014371593482792377, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.387024849653244, 0.0012950673699378967, 4.0595266909804195e-05, 0.0016053735744208097, 5.915556903346442e-05, 0.00023957251687534153, 2.72913839580724e-05, 0.004568422678858042, 0.0004761815653182566, 0.01562541350722313, 0.011085761711001396, 0.49522680044174194, 0.03870179131627083, 0.00026717083528637886, 0.0020083452109247446, 0.004042898770421743, 0.00032641590223647654, 0.0036371152382344007, 0.005066386889666319, 0.003996914252638817, 5.871349458175246e-06, 0.012405160814523697, 0.004702437669038773, 1.1574948075576685e-05, 0.0001442672364646569, 1.3832131116942037e-05, 7.560355879832059e-05, 1.5469962818315253e-05, 0.0020267819054424763, 3.810078851529397e-05, 0.003570300992578268, 0.001669161138124764, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4725484848022461, 0.0012542733456939459, 1.6021392639231635e-06, 0.007470476441085339, 2.098127151839435e-05, 0.00022495862504001707, 3.144747097394429e-06, 0.00015600249753333628, 5.6978915381478146e-05, 0.008746420964598656, 0.006501034367829561, 0.005835692398250103, 0.4674169719219208, 0.0013900346821174026, 0.0022286404855549335, 0.0015983363846316934, 0.0009148887475021183, 4.253075894666836e-05, 0.007089768536388874, 0.002937783719971776, 9.027125088323373e-06, 0.002526575466617942, 0.004196966998279095, 9.68642325460678e-07, 0.0005582970916293561, 1.858499126683455e-05, 8.944844012148678e-05, 6.846757969469763e-06, 7.68072932260111e-05, 1.4904732779541519e-05, 0.0030944147147238255, 0.0010494303423911333, 0.0019187491852790117, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14304865896701813, 0.0001688210031716153, 5.145543582329992e-07, 0.00013525491522159427, 0.001670985366217792, 0.000564366695471108, 4.049839844810776e-06, 4.1035786125576124e-05, 6.743645712958823e-07, 0.00026636559050530195, 3.54247895302251e-05, 0.00038403936196118593, 0.001104785711504519, 0.8484564423561096, 0.00022315295063890517, 0.00125994929112494, 2.182349817303475e-05, 4.0234138509731565e-07, 9.149295692623127e-06, 7.982308306964114e-05, 3.274861228419468e-05, 0.000818326196167618, 0.0002738100884016603, 6.036240307594198e-08, 5.658039754052879e-06, 0.0007130159065127373, 0.00026146037271246314, 3.5881919302482856e-06, 2.1928895876044407e-05, 4.159101507639207e-08, 0.00011863742111017928, 1.6391287545047817e-06, 3.8121448596939445e-05, 0.00023530120961368084, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.24328209459781647, 0.0009063901379704475, 0.0002717962197493762, 0.0012521950993686914, 3.105668383796001e-06, 0.001761557999998331, 7.013605227257358e-06, 0.040556132793426514, 0.0003538632008712739, 0.003045480465516448, 0.0016025315271690488, 0.0006016580737195909, 0.009119357913732529, 0.00086904939962551, 0.6830158829689026, 0.002626276109367609, 0.00010224885045317933, 7.45571151128388e-06, 3.991548510384746e-05, 0.00021531074889935553, 2.0861371012870222e-06, 0.000726094760466367, 0.0004009866388514638, 1.702951885818038e-05, 6.652241427218542e-05, 3.073251377827546e-07, 0.00018005711899604648, 5.3525491239270195e-06, 0.0071047646924853325, 2.1172911147004925e-05, 0.00044749100925400853, 5.941993367741816e-05, 3.588023173506372e-05, 0.001178023056127131, 0.00011552044452400878, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.24815866351127625, 0.0004487273108679801, 9.697681889520027e-06, 0.0010719134006649256, 7.626599654031452e-06, 0.00018865316815208644, 6.339164974633604e-05, 0.004560624714940786, 0.0010741765145212412, 0.015649856999516487, 0.002170345513150096, 0.0007331715896725655, 0.007097125519067049, 0.00044510516454465687, 0.016780385747551918, 0.6907860040664673, 0.00019302917644381523, 0.0003835348761640489, 0.0009248864953406155, 0.001862714416347444, 2.057407328948102e-07, 0.002524268114939332, 0.0015590769471600652, 3.3248329600610305e-07, 1.708475247141905e-05, 4.2712878212114447e-07, 1.1026169886463322e-05, 4.378564426588127e-06, 0.00015843719302210957, 1.5476502085221e-05, 0.0009950557723641396, 4.288491982151754e-05, 5.0623875722521916e-05, 0.000995215610601008, 1.3690095329366159e-05, 0.0010022438364103436, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15202470123767853, 0.0005997071275487542, 1.1835468285426032e-05, 0.10644718259572983, 9.087667422136292e-05, 0.0003228001296520233, 0.000542075838893652, 0.0006674103206023574, 0.00022456956503447145, 0.021209707483649254, 0.019741186872124672, 0.00012132841220591217, 0.015023781917989254, 0.0003238526696804911, 0.0003435326216276735, 0.003281805431470275, 0.6119738221168518, 0.00224235188215971, 0.008034387603402138, 0.0039560976438224316, 0.00143277901224792, 0.011238718405365944, 0.0009119117166846991, 1.6121168300742283e-05, 0.034571193158626556, 4.049971539643593e-05, 7.184680725913495e-05, 0.0005501421983353794, 0.0001563764235470444, 6.361229679896496e-06, 0.00147312565241009, 0.0009954210836440325, 1.2639467058761511e-05, 0.0010144341504201293, 3.702372487168759e-05, 5.788439739262685e-05, 0.00023058420629240572, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.035443101078271866, 2.0004812540719286e-05, 4.18805292667912e-08, 0.0003809734480455518, 5.874024395779998e-07, 2.792186933220364e-06, 2.425247657811269e-05, 8.103247819235548e-05, 0.00011144909512950107, 0.0027193769346922636, 0.00454584788531065, 4.083846943103708e-05, 0.0014098533429205418, 5.68071186535235e-07, 3.1567922178510344e-06, 8.66913323989138e-05, 0.0001365105708828196, 0.8487799763679504, 0.10322029143571854, 0.00040226749842986465, 2.244725919808843e-06, 0.0014588433550670743, 0.00012918819265905768, 6.671142216418957e-08, 7.290130452020094e-05, 3.444819185460801e-07, 1.4172115925248363e-06, 1.9379569494049065e-05, 4.5301294449018314e-05, 6.376938017638167e-06, 0.00026465157861821353, 0.0004582676920108497, 2.116967152687721e-05, 9.94484216789715e-05, 4.348984106172793e-08, 3.564729809113487e-07, 2.450852434776607e-06, 8.022138899832498e-06, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2318733036518097, 0.0003618353803176433, 8.543166529761947e-08, 0.006350396666675806, 1.0386469284640043e-06, 1.706758121144958e-05, 9.30641544982791e-06, 0.00014515469956677407, 9.0070599981118e-05, 0.01544187217950821, 0.04194100946187973, 8.23421487439191e-06, 0.022686535492539406, 5.4958513828751165e-06, 2.1405727238743566e-05, 5.323981531546451e-05, 0.0014441021485254169, 0.005718633532524109, 0.6479307413101196, 0.012128104455769062, 4.028283728985116e-05, 0.002520335605368018, 0.0007621925324201584, 6.570355992607801e-08, 0.0007668299949727952, 4.547823095890635e-07, 8.124479791149497e-06, 1.3828406736138277e-05, 3.526886212057434e-05, 9.165401934296824e-06, 0.0020918850786983967, 0.004286687355488539, 3.2024597658164566e-06, 0.0017294595018029213, 5.357091481528187e-07, 2.501844164726208e-06, 2.884956984416931e-06, 8.676738070789725e-05, 0.0014117915416136384, 0.0, 0.0, 0.0, 0.0], [0.3010721206665039, 0.0003067184588871896, 7.214547395051341e-07, 0.0010787947103381157, 3.671133526950143e-05, 9.958079317584634e-05, 2.7117800982523477e-06, 8.51582590257749e-05, 0.0006431526271626353, 0.00647402461618185, 0.0024339223746210337, 0.0026007171254605055, 0.012783723883330822, 0.0001047167825163342, 7.190534233814105e-05, 0.00023829020210541785, 0.00014132891374174505, 0.0010901358909904957, 0.006501613184809685, 0.6443015933036804, 0.0005654482520185411, 0.011171936988830566, 0.0017330633709207177, 3.266845851612743e-07, 7.267465116456151e-05, 1.726137998048216e-05, 5.252150003798306e-05, 2.8104516331950435e-06, 2.3941935069160536e-05, 6.633036537095904e-05, 0.002350717782974243, 0.00014581516734324396, 0.0007032529101707041, 0.0013972821179777384, 3.202367588528432e-05, 2.3009477445157245e-05, 2.68677049461985e-05, 1.7945263607543893e-05, 0.0001209545589517802, 0.001408109557814896, 0.0, 0.0, 0.0], [0.012865342199802399, 1.5182659808488097e-05, 2.972013524171757e-09, 0.0002875621721614152, 1.2528023489721818e-06, 1.252613128599478e-05, 4.250307028996758e-06, 8.040903594519477e-06, 3.2277103656497275e-08, 5.239708480075933e-05, 0.00013598549412563443, 1.002539192995755e-06, 5.5890355724841356e-05, 7.012691639829427e-06, 9.724845284608818e-08, 3.212419130704802e-07, 4.670393173000775e-05, 1.5471712231374113e-06, 1.7591864889254794e-05, 5.334270463208668e-05, 0.9805435538291931, 0.005806764122098684, 1.860846532508731e-05, 1.2037257857144823e-09, 3.0451696147792973e-05, 6.754650598850276e-07, 4.56516090707737e-06, 5.748487637902144e-06, 3.3398507639503805e-06, 7.534131096775809e-10, 7.689136509725358e-06, 3.901714080711827e-06, 4.307142731363456e-08, 5.192545700083429e-07, 5.106750791128434e-07, 8.534786211100709e-09, 1.0071753209217604e-08, 2.59315993389464e-06, 1.4836371065030107e-07, 3.5096545047963446e-07, 4.402238118927926e-06, 0.0, 0.0], [0.2408866286277771, 0.0003843706217594445, 3.8272382596460375e-08, 0.0007521198713220656, 4.429959517437965e-06, 1.3616883734357543e-05, 1.0370951713412069e-05, 0.00026089049060828984, 2.180447154387366e-06, 0.0008239771705120802, 0.0025784452445805073, 0.0001127499490394257, 0.001231508213095367, 3.884355010086438e-06, 3.183517401339486e-05, 0.00015905841428320855, 0.00016030347615014762, 0.00039731874130666256, 0.0017629768699407578, 0.001543086371384561, 0.0018108984222635627, 0.7393378019332886, 0.00752194132655859, 2.4824957645108725e-09, 2.4742143068579026e-05, 3.882949215494591e-07, 2.5054800971702207e-06, 3.208101816198905e-06, 4.296221959521063e-05, 2.8116325268001674e-08, 1.3948349078418687e-05, 1.6366040654247627e-05, 6.464315902121598e-06, 3.004602695000358e-05, 1.3071932869479497e-07, 9.608824029783136e-07, 1.787801124919497e-06, 1.753078095134697e-06, 4.386996351968264e-06, 1.6937889085966162e-05, 1.7175851098727435e-05, 2.5725426894496195e-05, 0.0], [0.5535919070243835, 0.17262260615825653, 5.0000453484244645e-05, 0.01042777020484209, 8.65629335748963e-05, 0.0002456592337694019, 5.955337837804109e-05, 0.001045247190631926, 0.0008384220418520272, 0.01357327215373516, 0.011254074051976204, 0.003105845768004656, 0.02095966413617134, 0.0010894862934947014, 0.0004093680181540549, 0.002362574217841029, 0.0016563499812036753, 0.0009357953676953912, 0.0033135719131678343, 0.00543051166459918, 0.0010150577872991562, 0.032745711505413055, 0.1570950448513031, 3.374255175003782e-05, 0.0007222488638944924, 2.940950980701018e-05, 6.13709562458098e-05, 3.093114719376899e-05, 0.00024038334959186614, 4.459061528905295e-05, 0.0006923413602635264, 0.00030414367211051285, 0.0001727685157675296, 0.0012000406859442592, 0.00010696896788431332, 2.8368263883749023e-05, 1.842521487560589e-05, 2.8223597837495618e-05, 3.744996865862049e-05, 0.00011934195208596066, 0.00018961371097248048, 3.498619480524212e-05, 0.0019905013032257557]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9940896034240723, 0.0059104301035404205, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9918997287750244, 0.002507372060790658, 0.005592965055257082, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.99346524477005, 0.003521271515637636, 0.00027059370768256485, 0.0027428686153143644, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9289718270301819, 0.003740765620023012, 0.0001203682622872293, 0.002464901190251112, 0.06470214575529099, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9885430335998535, 0.0006141006015241146, 0.00041145377326756716, 0.007392834406346083, 0.000539508939255029, 0.002499047899618745, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9694589376449585, 0.0009395341039635241, 2.8261547413421795e-05, 0.0020353952422738075, 2.71175304078497e-05, 0.00829475186765194, 0.019215956330299377, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9474747776985168, 0.005424594972282648, 0.004983103834092617, 0.021659083664417267, 1.2853307453042362e-05, 0.0011076473165303469, 0.001525420811958611, 0.01781250163912773, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9157270193099976, 0.006453890353441238, 0.0011921703116968274, 0.04077358543872833, 3.2812098652357236e-05, 0.002033841796219349, 0.00022136607731226832, 0.010999627411365509, 0.022565681487321854, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8299553990364075, 0.0021487579215317965, 0.002185288118198514, 0.04418088495731354, 1.725433139654342e-05, 0.0014081649715080857, 0.020800573751330376, 0.08762412518262863, 0.000955707801040262, 0.010723880492150784, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9581490159034729, 0.004779844544827938, 0.00035515165654942393, 0.0029918320942670107, 0.0032907268032431602, 0.007958430796861649, 0.0017856257036328316, 0.013284464366734028, 0.00015784945571795106, 0.0009225388057529926, 0.0063244919292628765, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.944205641746521, 0.0028841965831816196, 4.829783574678004e-05, 0.007261567283421755, 0.000468916492536664, 0.005381070543080568, 0.00017496546206530184, 0.006957130506634712, 0.00012532716209534556, 0.004074090626090765, 0.016885550692677498, 0.011533298529684544, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7170465588569641, 0.0029263142496347427, 1.372535371046979e-05, 0.0010318169370293617, 0.1216491237282753, 0.14445394277572632, 0.0009355274378322065, 0.005332366097718477, 4.7773132791917305e-06, 0.0012480594450607896, 0.00030511102522723377, 0.0019744164310395718, 0.0030781622044742107, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7894357442855835, 0.0020973754581063986, 0.0020318119786679745, 0.010099953971803188, 0.00036911203642375767, 0.06713488698005676, 0.001530290232039988, 0.08991599082946777, 0.0002695393341127783, 0.005345543846487999, 0.005062697920948267, 0.0006220968207344413, 0.019640257582068443, 0.006444734986871481, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8822957873344421, 0.0018603845965117216, 0.00040732327033765614, 0.003660863032564521, 0.0005736579769290984, 0.007988521829247475, 0.02257135882973671, 0.02860121801495552, 0.0003292130131740123, 0.0042566233314573765, 0.006161138415336609, 0.0010443199425935745, 0.02063308097422123, 0.005851183086633682, 0.013765350915491581, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.786965548992157, 0.0042131864465773106, 0.0001677500840742141, 0.06983140110969543, 0.002818087814375758, 0.007715716026723385, 0.031199535354971886, 0.017051437869668007, 0.00018332427134737372, 0.012005697004497051, 0.017662225291132927, 0.0002890100877266377, 0.03032497689127922, 0.004889386706054211, 0.003095801919698715, 0.011587009765207767, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9286779761314392, 0.0015812769997864962, 1.4040829228179064e-06, 0.0019428395899012685, 1.1035229363187682e-05, 0.0002707966777961701, 0.002038027858361602, 0.0036648765671998262, 0.00022354313114192337, 0.0007879888289608061, 0.04505370557308197, 0.00048093145596794784, 0.010849039070308208, 3.6126039049122483e-06, 6.594638398382813e-05, 0.0008642339380457997, 0.0034828479401767254, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8384721875190735, 0.0026208001654595137, 3.342468335176818e-05, 0.01232653483748436, 7.991644088178873e-05, 0.003431107848882675, 0.0022304044105112553, 0.008115601725876331, 0.0006596868042834103, 0.008109694346785545, 0.0638236477971077, 0.00041881605284288526, 0.03681996464729309, 0.00012104758206987754, 0.0005988333723507822, 0.00862804614007473, 0.003624672070145607, 0.00988560076802969, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8221591114997864, 0.0057971584610641, 0.00019727385370060802, 0.005305740050971508, 0.0009490191005170345, 0.01202884316444397, 0.00190554594155401, 0.0023991470225155354, 0.01059637125581503, 0.014539752155542374, 0.011935291811823845, 0.01170417945832014, 0.048201464116573334, 0.006271726917475462, 0.006691561546176672, 0.02466079778969288, 0.005656811874359846, 0.005013483576476574, 0.003986795898526907, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9537661671638489, 0.0015298646176233888, 5.0040102905768435e-06, 0.007204053923487663, 0.0002875779173336923, 0.0017434227047488093, 0.00982952956110239, 0.004322257358580828, 3.13615760205721e-06, 0.00010985424887621775, 0.008465062826871872, 0.0003484692715574056, 0.0013010905822739005, 0.0008953196229413152, 6.606405804632232e-05, 5.8244528190698475e-05, 0.0037666442804038525, 0.0005685480427928269, 0.001456835656426847, 0.004272699821740389, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9650986790657043, 0.0014334715669974685, 4.691014055424603e-06, 0.0012125533539801836, 0.0003355584340170026, 0.0012382488930597901, 0.0013702923897653818, 0.002564979949966073, 6.760245128134557e-07, 7.471856224583462e-05, 0.001875573769211769, 0.0005754472804255784, 0.0028297884855419397, 0.0007055316236801445, 0.00027714675525203347, 0.00048612672253511846, 0.001569450250826776, 0.0003992257115896791, 0.0028444859199225903, 0.004788658116012812, 0.010314569808542728, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8593109250068665, 0.020267227664589882, 0.002777141286060214, 0.013781089335680008, 0.00030035056988708675, 0.002284612273797393, 0.0010426462395116687, 0.009922247380018234, 0.003133647609502077, 0.007885495200753212, 0.009642801247537136, 0.001854671980254352, 0.00787472166121006, 0.0011870941380038857, 0.002824161434546113, 0.00242280843667686, 0.004221721086651087, 0.004099525045603514, 0.006750837899744511, 0.0058750067837536335, 0.002170956926420331, 0.030370241031050682, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10544665902853012, 0.01270485483109951, 0.7639007568359375, 0.08842571824789047, 0.00018797894881572574, 0.0002628951915539801, 0.0007833988056518137, 0.001709726988337934, 0.002409537322819233, 0.014530741609632969, 0.0006739442469552159, 3.582143835956231e-05, 0.0008068872266449034, 0.000150876454426907, 0.001663832925260067, 0.0002909536415245384, 0.001969669247046113, 2.807584314723499e-05, 8.648991206428036e-05, 0.0001831684639910236, 5.4813881433801726e-05, 0.0004074028111062944, 0.003285784274339676, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.33122679591178894, 0.007398602087050676, 0.006046590860933065, 0.38090354204177856, 0.0007559282821603119, 0.003547786269336939, 0.00850890576839447, 0.02200973406434059, 0.0006401475402526557, 0.06486588716506958, 0.012087974697351456, 2.1607689632219262e-05, 0.004766723141074181, 0.0004923217929899693, 0.002488807076588273, 0.00041253515519201756, 0.11648410558700562, 0.00017437727365177125, 0.001129980431869626, 0.0012968714581802487, 0.001893762033432722, 0.006374640390276909, 0.012117794714868069, 0.014354660175740719, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.022601181641221046, 0.0004386410873848945, 2.7166800009581493e-06, 0.0003694462648127228, 0.9364854693412781, 0.032159216701984406, 0.00016640119429212064, 0.00037730683106929064, 2.719356189118116e-07, 9.796216181712225e-05, 2.4832312192302197e-05, 0.00014765629020985216, 0.0004301169828977436, 0.004391680471599102, 0.0003911138919647783, 5.064404831500724e-05, 0.00018704244575928897, 6.401351129170507e-06, 2.0583942387020215e-05, 0.0002504437288735062, 4.058926060679369e-05, 0.0006710129673592746, 0.0006265917327255011, 2.0016514099552296e-06, 6.063750697649084e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.039748288691043854, 0.0016230319160968065, 2.9864083899155958e-06, 0.0004959744983352721, 0.0075991530902683735, 0.903511106967926, 0.00030855785007588565, 0.020541049540042877, 3.7260215322021395e-06, 0.0009059454896487296, 0.0005234951386228204, 0.00019036592857446522, 0.002769604790955782, 0.001758625265210867, 0.006682183127850294, 0.0016490903217345476, 0.00027723185485228896, 2.3619986677658744e-05, 0.0003351525228936225, 0.001790589652955532, 0.00015660120698157698, 0.0019276170060038567, 0.0019798888824880123, 1.2426404509824351e-06, 5.812747258460149e-05, 0.005136688239872456, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.017394619062542915, 8.34300517453812e-05, 1.299515815844643e-06, 0.0006728374864906073, 1.4733542229805607e-05, 0.00010638036474119872, 0.9528148770332336, 0.021972037851810455, 7.827152899153589e-07, 1.5493780665565282e-05, 0.0031969614792615175, 3.257389471400529e-05, 0.0001382468908559531, 2.389342444075737e-05, 0.000840842432808131, 0.00010894631850533187, 0.00028938756440766156, 0.0001089509969460778, 6.79441072861664e-05, 0.00012526210048235953, 0.00019726519531104714, 0.0008631302043795586, 0.0002980587596539408, 2.3374896045424975e-06, 0.0003466096241027117, 2.3777269234415144e-05, 0.00025927400565706193, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05558102950453758, 0.0006445589242503047, 4.0934705225481594e-07, 0.00042194841080345213, 1.340792550763581e-06, 0.000646521570160985, 0.0018358062952756882, 0.9107965230941772, 6.180139735079138e-06, 0.00014045907300896943, 0.010802187956869602, 0.0002091953792842105, 0.0011495650978758931, 1.2848360711359419e-05, 0.003265448845922947, 0.0001199487887788564, 0.00020614646200556308, 0.0006146047380752861, 0.0012190378038212657, 0.001267205341719091, 0.00038551114266738296, 0.0035668814089149237, 0.0011672672117128968, 5.425858375929238e-07, 0.0003176873433403671, 1.1132586905659991e-06, 0.002803834853693843, 0.002816123189404607, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08360803127288818, 0.0049411761574447155, 5.052056076237932e-05, 0.007228298112750053, 1.1925421858904883e-06, 0.000162318188813515, 0.0002728418621700257, 0.005935708060860634, 0.6047206521034241, 0.26288917660713196, 0.005585616920143366, 7.11743050487712e-05, 0.0013994898181408644, 2.673692279131501e-06, 0.0015110736712813377, 9.4614464615006e-05, 0.0003633210726547986, 0.0007099778158590198, 0.0009741897229105234, 0.003578288946300745, 1.0970221410389058e-05, 0.0001563943224027753, 0.003689345670863986, 0.0002846125280484557, 0.004334022291004658, 1.3892353081246256e-06, 0.00024127897631842643, 0.0001852042623795569, 0.006996475625783205, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1152101382613182, 0.006714800372719765, 0.000162112366524525, 0.0255129411816597, 6.206613761605695e-06, 0.0004051269206684083, 5.4560387070523575e-05, 0.00351078063249588, 0.014202400110661983, 0.7775508165359497, 0.0034905581269413233, 5.7819983339868486e-05, 0.0047684297896921635, 3.606372411013581e-05, 0.0007714288076385856, 0.0003429505741223693, 0.002843409776687622, 9.518005390418693e-05, 0.003136677900329232, 0.005728919059038162, 4.441967757884413e-05, 0.0004939687205478549, 0.008516553789377213, 0.0005445042625069618, 0.014522203244268894, 5.894724836252863e-06, 0.00030799367232248187, 2.679031968000345e-05, 0.0010097086196765304, 0.009926472790539265, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.03713580593466759, 0.0008775312453508377, 7.848548557376489e-05, 0.015299011021852493, 1.4544053783538402e-06, 0.00010877755266847089, 0.006165174767374992, 0.014768626540899277, 7.875503069953993e-05, 0.001974995480850339, 0.845594584941864, 7.480713975382969e-05, 0.0028255584184080362, 5.115834369462391e-07, 0.00017026669229380786, 0.00015173685096669942, 0.006004323251545429, 0.0034732050262391567, 0.012612894177436829, 0.0006110360845923424, 0.0005419608787633479, 0.0009975193534046412, 0.002906821435317397, 0.0003956863656640053, 0.020268796011805534, 1.613207473383227e-06, 0.00021117184951435775, 0.007802305277436972, 0.017504550516605377, 2.8824499167967588e-05, 0.0013330691726878285, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04171165078878403, 0.0007273957598954439, 5.084281838207971e-06, 0.0004205020668450743, 0.0002682959020603448, 0.00036437364178709686, 0.00021653271687682718, 0.0014350030105561018, 5.140026132721687e-06, 9.150908590527251e-05, 0.0012519678566604853, 0.9164962768554688, 0.025602256879210472, 0.0013075522147119045, 0.0030348259024322033, 0.0005533328512683511, 0.00034456001594662666, 0.00023525662254542112, 0.0004293296951800585, 0.00137053313665092, 3.670702062663622e-05, 0.0015910869697108865, 0.00048218751908279955, 9.56441908783745e-06, 0.00015524028276558965, 0.00017542645218782127, 0.0004897234030067921, 6.953987031010911e-05, 0.0008855744963511825, 1.1304659892630298e-06, 4.837113374378532e-05, 0.0001840855402406305, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1554105579853058, 0.0020603404846042395, 5.904707904846873e-06, 0.005144172348082066, 0.00025823267060332, 0.0014501075493171811, 5.391828017309308e-05, 0.0021640402264893055, 2.1527968783630058e-05, 0.0016064351657405496, 0.010191709734499454, 0.010151085443794727, 0.7491754293441772, 0.0027421838603913784, 0.01017574779689312, 0.005842445883899927, 0.0030403982382267714, 3.083394767600112e-05, 0.006112964358180761, 0.016732651740312576, 0.00015879959391895682, 0.0020674674306064844, 0.0016374929109588265, 1.1047049156331923e-05, 0.0014025868149474263, 0.0001992537872865796, 0.001220017671585083, 2.1487703634193167e-05, 0.0007244196604005992, 5.2953896556573454e-06, 0.0011675384594127536, 0.0018881573341786861, 0.0071258689276874065, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.03857047110795975, 0.00048676287406124175, 8.812001084379517e-08, 0.000122779980301857, 0.005581730045378208, 0.0025055643636733294, 2.1938227291684598e-05, 0.00011670660751406103, 8.340171575582644e-08, 6.456951086875051e-05, 1.648088073125109e-05, 0.00033052131766453385, 0.0009664044482633471, 0.9410394430160522, 0.0025494787842035294, 0.000986628350801766, 0.00023048926959745586, 2.1702271624235436e-07, 9.28598819882609e-06, 0.001075997482985258, 8.328325202455744e-05, 0.0006203338853083551, 0.0003860929573420435, 1.7099183935442852e-07, 1.0056071005237754e-05, 0.0019989400170743465, 0.0020207997877150774, 4.052772510476643e-06, 1.9309491108288057e-05, 3.980539009518225e-09, 1.3065056009509135e-05, 4.567590394799481e-07, 5.0416052545187995e-05, 0.00011722340423148125, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.32133859395980835, 0.0015032502124086022, 0.00035717309219762683, 0.009410788305103779, 0.00016890434199012816, 0.016028327867388725, 0.0002053897042060271, 0.019911326467990875, 6.106824002927169e-05, 0.0025995566975325346, 0.0018900673603639007, 0.0003402142319828272, 0.018113745376467705, 0.0032540573738515377, 0.5562437772750854, 0.008188020437955856, 0.0034578992053866386, 2.759976268862374e-05, 0.00021329266019165516, 0.004111619666218758, 7.46081059332937e-05, 0.0024103508330881596, 0.0031445701606571674, 0.0002873913326766342, 0.0028005591593682766, 5.076818706584163e-05, 0.010558384470641613, 8.335074380738661e-05, 0.007083627860993147, 1.715210783004295e-05, 0.0007566043641418219, 0.00013426884834188968, 4.393113704281859e-05, 0.0036519141867756844, 0.0014778183540329337, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.16036191582679749, 0.0012780018150806427, 2.2185913621797226e-05, 0.0023713421542197466, 0.0002248762029921636, 0.001783479005098343, 0.003998146392405033, 0.00463333074003458, 4.07644038205035e-05, 0.0015416644746437669, 0.0015637290198355913, 0.00047258863924071193, 0.01683974452316761, 0.0034527594689279795, 0.017767908051609993, 0.7616898417472839, 0.004626939073204994, 0.00039177873986773193, 0.000968358654063195, 0.0050940741784870625, 3.4372260415693745e-06, 0.0016141458181664348, 0.0016999210929498076, 1.1049841305066366e-05, 0.00029084301786497235, 4.1652379877632484e-05, 0.000830529781524092, 0.00043556030141189694, 0.0004473649896681309, 1.989761585718952e-06, 0.0003503866319078952, 6.375126395141706e-05, 6.234755710465834e-05, 0.0025966004468500614, 0.0005663724150508642, 0.00186053360812366, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.16436104476451874, 0.00263788178563118, 4.14170281146653e-05, 0.06607695668935776, 0.0011643714969977736, 0.0015624640509486198, 0.01255388930439949, 0.0027841543778777122, 3.6274232115829363e-05, 0.004333572927862406, 0.006344497669488192, 0.00013702016440220177, 0.022220438346266747, 0.001139097730629146, 0.0015757762594148517, 0.003894534194841981, 0.5750948190689087, 0.0019101364305242896, 0.004216517321765423, 0.005103054456412792, 0.0011178136337548494, 0.011998940259218216, 0.006141388788819313, 8.207879727706313e-05, 0.0696118101477623, 0.0019772748928517103, 0.0024057570844888687, 0.01315633486956358, 0.0013538515195250511, 9.255067197955213e-06, 0.003937660250812769, 0.0014234132831916213, 5.669038000633009e-05, 0.005460998509079218, 0.0004025489615742117, 0.0006155924638733268, 0.003060702932998538, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07928115874528885, 0.0006841905415058136, 6.815574948859648e-08, 0.0010951507138088346, 2.0406362182257e-06, 2.2378419089363888e-05, 0.0006314062047749758, 0.0003747308801393956, 2.3558166503789835e-05, 0.00018500992155168205, 0.008331519551575184, 7.647382153663784e-05, 0.00418297341093421, 2.444711526550236e-07, 1.1477874068077654e-05, 5.1371527661103755e-05, 0.0018863201839849353, 0.6889944076538086, 0.1946372091770172, 0.004766413476318121, 4.9866932386066765e-05, 0.0028069806285202503, 0.005558052100241184, 3.95493913174505e-07, 0.0008065987494774163, 3.353459078425658e-06, 5.2719624363817275e-05, 0.0003899818693753332, 0.0004888054681941867, 7.204272606031736e-06, 9.927168866852298e-05, 0.003168433904647827, 3.9500540879089385e-05, 0.0009883888997137547, 7.518784883586704e-08, 1.4315326097857906e-06, 6.484955520136282e-06, 0.00029431298025883734, 0.0, 0.0, 0.0, 0.0, 0.0], [0.28373828530311584, 0.004487877711653709, 5.058388069301145e-06, 0.019317252561450005, 5.4460702813230455e-05, 0.0012376051163300872, 0.0017604499589651823, 0.0063126953318715096, 0.00024137768195942044, 0.006789366248995066, 0.07160750776529312, 0.0003089959791395813, 0.06864359229803085, 4.332327443989925e-05, 0.0004192555497866124, 0.0020601244177669287, 0.006795847322791815, 0.01733478344976902, 0.33858922123908997, 0.03910181671380997, 0.0007660706178285182, 0.009993083775043488, 0.007829555310308933, 1.3115719411871396e-05, 0.01356492843478918, 6.270289304666221e-05, 0.0021037741098552942, 0.0009936662390828133, 0.00333457812666893, 9.826785390032455e-05, 0.006462769117206335, 0.03356613591313362, 0.00019831513054668903, 0.030609816312789917, 1.5713876564404927e-05, 0.0001204574218718335, 0.0005064337747171521, 0.001992761390283704, 0.018918951973319054, 0.0, 0.0, 0.0, 0.0], [0.25413063168525696, 0.007289606146514416, 1.6370242519769818e-05, 0.005644946824759245, 0.00047911194269545376, 0.003032122040167451, 0.0008039328386075795, 0.0005737085011787713, 0.001770934322848916, 0.0052279396913945675, 0.006573305930942297, 0.0037488588131964207, 0.039559945464134216, 0.0013069461565464735, 0.002049593720585108, 0.0032922588288784027, 0.005070534534752369, 0.0043929279781877995, 0.0029604455921798944, 0.5652295351028442, 0.006637280806899071, 0.020123612135648727, 0.013719636015594006, 3.509566158754751e-05, 0.00249256263487041, 0.0006804370786994696, 0.0050042420625686646, 0.0004963083192706108, 0.0002992783847730607, 0.0005683296476490796, 0.006156585179269314, 0.0014163292944431305, 0.002080241683870554, 0.01701081357896328, 0.0011587258195504546, 0.0008447651634924114, 0.0012247745180502534, 0.002057015197351575, 0.0027713908348232508, 0.0020689249504357576, 0.0, 0.0, 0.0], [0.02277885377407074, 0.00018681293295230716, 1.6159448890107342e-08, 0.0006753625930286944, 3.9720243876217864e-06, 1.8965230992762372e-05, 0.00014574530359823257, 0.00015363575948867947, 1.437089469646935e-08, 3.714960257639177e-06, 0.00038176553789526224, 9.189574484480545e-06, 0.00013344796025194228, 1.550033084640745e-05, 4.871678356721532e-06, 4.94858227284567e-07, 0.0004079081991221756, 1.881936623249203e-05, 3.495846613077447e-05, 0.00018379271205049008, 0.8935850262641907, 0.08001779019832611, 0.0004949963185936213, 6.95558668439844e-08, 0.0003611761494539678, 4.126275143789826e-06, 4.2135678086197004e-05, 0.0001535058399895206, 9.319555101683363e-05, 8.534826179129595e-10, 9.503900741947291e-07, 2.6511430405662395e-05, 9.136761605077481e-07, 4.42588907390018e-06, 3.6229419038136257e-06, 4.963176820638182e-07, 6.715119571509831e-09, 3.286409264546819e-05, 3.7280919968907256e-06, 3.7352272101998096e-06, 1.2850606253778096e-05, 0.0, 0.0], [0.04306096211075783, 0.00043120133341290057, 4.85736819655358e-08, 0.00029497192008420825, 6.446816769312136e-06, 1.794057789084036e-05, 5.2368359320098534e-05, 0.00012123115448048338, 8.686286356862638e-09, 4.085900854988722e-06, 0.00020099191169720143, 2.1525078409467824e-05, 0.0004682287108153105, 1.1544256267370656e-05, 3.74926021322608e-05, 7.720605935901403e-06, 0.0005387069541029632, 8.118282858049497e-05, 0.0002466407313477248, 0.0005759781342931092, 0.0023978566750884056, 0.9488213062286377, 0.002120599150657654, 3.640034407226267e-08, 7.682813156861812e-05, 2.893220880650915e-06, 1.444738245481858e-05, 1.2731605238514021e-05, 2.7991360184387304e-05, 3.7476780012646316e-10, 5.714333610740141e-07, 1.0850835678866133e-05, 3.7292174965841696e-06, 4.0900900785345584e-05, 3.826700321951648e-06, 6.126327207311988e-06, 5.169480346012278e-07, 4.086086119059473e-05, 1.2315524145378731e-05, 2.700609366002027e-05, 3.1881831091595814e-05, 0.00016747090558055788, 0.0], [0.26228708028793335, 0.04815318435430527, 0.0006689673755317926, 0.021233247593045235, 9.166418021777645e-05, 0.00039222234045155346, 0.00020396779291331768, 0.0021344097331166267, 0.001094849780201912, 0.004910758230835199, 0.006146768573671579, 0.0005123682785779238, 0.009844929911196232, 0.0001430303673259914, 0.0010762682650238276, 0.00034581031650304794, 0.010050452314317226, 0.008435730822384357, 0.008105281740427017, 0.00602466706186533, 0.0038135156501084566, 0.04830698296427727, 0.49913153052330017, 0.0012536782305687666, 0.033902086317539215, 0.00010517781629459932, 0.00037178199272602797, 8.200502634281293e-05, 0.0012060013832524419, 0.0001819609315134585, 0.0013505015522241592, 0.0009115547291003168, 0.00011081639240728691, 0.0016774724936112761, 3.677561107906513e-05, 0.00017677698633633554, 4.57889873359818e-05, 0.00101330759935081, 0.0026200786232948303, 0.002004811307415366, 0.0010430817492306232, 0.00036673821159638464, 0.008431942202150822]]], "attentionHeadNames": ["L6H9", "L5H5"], "tokens": ["<|endoftext|>", "Research", " in", " mechan", "istic", " interpret", "ability", " seeks", " to", " explain", " behaviors", " of", " machine", " learning", " models", " in", " terms", " of", " their", " internal", " components", ".", "Research", " in", " mechan", "istic", " interpret", "ability", " seeks", " to", " explain", " behaviors", " of", " machine", " learning", " models", " in", " terms", " of", " their", " internal", " components", "."]}
     )
@@ -2086,9 +2086,9 @@ <h4>Implications<a class="headerlink" href="#Implications" title="Permalink to t
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="8a472635-6d79-4701-854f-c62e62ebce54" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("8a472635-6d79-4701-854f-c62e62ebce54")) {                    Plotly.newPlot(                        "8a472635-6d79-4701-854f-c62e62ebce54",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.03906984627246857,0.0004489120910875499,0.03133600950241089,0.007519586011767387,0.034592170268297195,0.0003623008378781378,0.03451276570558548,0.19740214943885803,0.03844786807894707,0.04053793475031853,0.027628779411315918,0.02496313489973545],[0.18906497955322266,0.1721990704536438,0.06807752698659897,0.044945165514945984,0.0790855661034584,0.03096737712621689,0.028282105922698975,0.036443259567022324,0.026936713606119156,0.018826233223080635,0.04510088637471199,0.0065726605243980885],[0.15745531022548676,0.020724497735500336,0.48180025815963745,0.2991351783275604,0.1076490506529808,0.33004093170166016,0.09975510090589523,0.04926132783293724,0.2549382448196411,0.3606453239917755,0.12571798264980316,0.07931824773550034],[0.0058439951390028,0.1578730344772339,0.4189079701900482,0.30129143595695496,0.014345025643706322,0.03234415501356125,0.3312898576259613,0.5285982489585876,0.34241825342178345,0.10183659940958023,0.10516025871038437,0.22331231832504272],[0.10626537352800369,0.11930901557207108,0.02288069576025009,0.22826972603797913,0.02000402845442295,0.10010094195604324,0.17392224073410034,0.17407013475894928,0.025876855477690697,0.10249904543161392,0.009514803998172283,0.9921424388885498],[0.019766751676797867,0.005283255595713854,0.166484072804451,0.12087711691856384,0.1649997979402542,0.008032665587961674,0.4176996946334839,0.02582792565226555,0.04802580177783966,0.016231726855039597,0.031101685017347336,0.024261072278022766],[0.21729078888893127,0.03909975662827492,0.018048642203211784,0.059900619089603424,0.0329345278441906,0.0873454138636589,0.026895388960838318,0.09439495950937271,0.49926066398620605,0.00624012341722846,0.027026763185858727,0.1278562843799591],[0.25116443634033203,0.013308697380125523,0.006663368083536625,0.03743036091327667,0.02331540174782276,0.017407197505235672,0.02206745184957981,0.022141238674521446,0.04502354562282562,0.02084278129041195,0.008310752920806408,0.017167769372463226],[0.020890571177005768,0.016537854447960854,0.021583056077361107,0.015005743131041527,0.024211907759308815,0.1019899770617485,0.029100272804498672,0.22793740034103394,0.02781473658978939,0.01794101856648922,0.02482905425131321,0.03806224465370178],[0.02607565000653267,0.015407402068376541,0.020444145426154137,0.14558184146881104,0.01247023232281208,0.017151616513729095,0.013311783783137798,0.02445186860859394,0.018111703917384148,0.013193286024034023,0.03573964536190033,0.0187982190400362],[0.02147809788584709,0.018419133499264717,0.0181836299598217,0.021721404045820236,0.03156769275665283,0.03470622003078461,0.017550060525536537,0.011417530477046967,0.015795614570379257,0.04592300206422806,0.016215480864048004,0.030394626781344414],[0.033205095678567886,0.017571475356817245,0.015131049789488316,0.04148414358496666,0.015181177295744419,0.01758996769785881,0.01514849066734314,0.017676066607236862,0.06622707098722458,0.01845112442970276,0.01700744405388832,0.02974928542971611]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Previous Token Scores"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="c9a93815-e882-486d-9f08-66c767cc190d" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("c9a93815-e882-486d-9f08-66c767cc190d")) {                    Plotly.newPlot(                        "c9a93815-e882-486d-9f08-66c767cc190d",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.03906984627246857,0.0004489120910875499,0.03133600950241089,0.007519586011767387,0.034592170268297195,0.0003623008378781378,0.03451276570558548,0.19740214943885803,0.03844786807894707,0.04053793475031853,0.027628779411315918,0.02496313489973545],[0.18906497955322266,0.1721990704536438,0.06807752698659897,0.044945165514945984,0.0790855661034584,0.03096737712621689,0.028282105922698975,0.036443259567022324,0.026936713606119156,0.018826233223080635,0.04510088637471199,0.0065726605243980885],[0.15745531022548676,0.020724497735500336,0.48180025815963745,0.2991351783275604,0.1076490506529808,0.33004093170166016,0.09975510090589523,0.04926132783293724,0.2549382448196411,0.3606453239917755,0.12571798264980316,0.07931824773550034],[0.0058439951390028,0.1578730344772339,0.4189079701900482,0.30129143595695496,0.014345025643706322,0.03234415501356125,0.3312898576259613,0.5285982489585876,0.34241825342178345,0.10183659940958023,0.10516025871038437,0.22331231832504272],[0.10626537352800369,0.11930901557207108,0.02288069576025009,0.22826972603797913,0.02000402845442295,0.10010094195604324,0.17392224073410034,0.17407013475894928,0.025876855477690697,0.10249904543161392,0.009514803998172283,0.9921424388885498],[0.019766751676797867,0.005283255595713854,0.166484072804451,0.12087711691856384,0.1649997979402542,0.008032665587961674,0.4176996946334839,0.02582792565226555,0.04802580177783966,0.016231726855039597,0.031101685017347336,0.024261072278022766],[0.21729078888893127,0.03909975662827492,0.018048642203211784,0.059900619089603424,0.0329345278441906,0.0873454138636589,0.026895388960838318,0.09439495950937271,0.49926066398620605,0.00624012341722846,0.027026763185858727,0.1278562843799591],[0.25116443634033203,0.013308697380125523,0.006663368083536625,0.03743036091327667,0.02331540174782276,0.017407197505235672,0.02206745184957981,0.022141238674521446,0.04502354562282562,0.02084278129041195,0.008310752920806408,0.017167769372463226],[0.020890571177005768,0.016537854447960854,0.021583056077361107,0.015005743131041527,0.024211907759308815,0.1019899770617485,0.029100272804498672,0.22793740034103394,0.02781473658978939,0.01794101856648922,0.02482905425131321,0.03806224465370178],[0.02607565000653267,0.015407402068376541,0.020444145426154137,0.14558184146881104,0.01247023232281208,0.017151616513729095,0.013311783783137798,0.02445186860859394,0.018111703917384148,0.013193286024034023,0.03573964536190033,0.0187982190400362],[0.02147809788584709,0.018419133499264717,0.0181836299598217,0.021721404045820236,0.03156769275665283,0.03470622003078461,0.017550060525536537,0.011417530477046967,0.015795614570379257,0.04592300206422806,0.016215480864048004,0.030394626781344414],[0.033205095678567886,0.017571475356817245,0.015131049789488316,0.04148414358496666,0.015181177295744419,0.01758996769785881,0.01514849066734314,0.017676066607236862,0.06622707098722458,0.01845112442970276,0.01700744405388832,0.02974928542971611]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Previous Token Scores"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('8a472635-6d79-4701-854f-c62e62ebce54');
+var gd = document.getElementById('c9a93815-e882-486d-9f08-66c767cc190d');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2116,9 +2116,9 @@ <h4>Implications<a class="headerlink" href="#Implications" title="Permalink to t
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="fae2e193-59a0-4811-84ef-98e7b34809aa" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("fae2e193-59a0-4811-84ef-98e7b34809aa")) {                    Plotly.newPlot(                        "fae2e193-59a0-4811-84ef-98e7b34809aa",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.003192325122654438,0.1323632299900055,0.005006915889680386,1.0427492270537186e-05,0.0013110269792377949,0.7034571170806885,0.004262051545083523,0.0001649669575272128,0.0024746304843574762,0.0008572910446673632,0.018891464918851852,0.008690940216183662],[0.0002916363882832229,0.00013781932648271322,0.0015036164550110698,0.005392488092184067,0.001858389819972217,0.009062976576387882,0.012414469383656979,0.00224055303260684,0.005135663319379091,0.005220625549554825,0.005546468310058117,0.029750702902674675],[0.0024816489312797785,0.009442185051739216,0.00034561342909000814,0.00025915156584233046,0.005211671814322472,0.0005709662800654769,0.0015209924895316362,0.006313098594546318,0.001560926903039217,0.00042158187716268003,0.0001535943301860243,0.005160389002412558],[0.6775667071342468,0.002840489149093628,0.0007841649930924177,0.004712563939392567,0.006322908215224743,0.00620671221986413,0.0005474375793710351,0.0003782894345931709,0.0020156530663371086,0.007952774874866009,0.003576836548745632,0.0026087891310453415],[0.008604029193520546,0.0070285736583173275,0.00759880430996418,0.0034427959471940994,0.016561321914196014,0.005979627370834351,0.0048697832971811295,0.0007624494028277695,0.006062139756977558,0.007536678109318018,0.012022883631289005,1.055450974014882e-12],[0.00950299110263586,0.008562120608985424,0.0041626025922596455,0.003008681582286954,0.006847443990409374,0.004358135629445314,0.007669272366911173,0.009584203362464905,0.007618824020028114,0.004328039940446615,0.04140293970704079,0.009761848486959934],[0.00445613032206893,0.008873283863067627,0.007405218668282032,0.006224926561117172,0.007319129537791014,0.005623883102089167,0.01734965480864048,0.005529423244297504,0.0029201731085777283,0.008636709302663803,0.006222232710570097,0.008358956314623356],[0.003699968568980694,0.041079357266426086,0.04148266464471817,0.009313643909990788,0.009097038768231869,0.008774377405643463,0.007298501208424568,0.023312222212553024,0.00884333811700344,0.00987985823303461,0.017598574981093407,0.006039811763912439],[0.008986336179077625,0.028667127713561058,0.00889119878411293,0.010114572942256927,0.009737403132021427,0.00761164166033268,0.009763195179402828,0.005155565217137337,0.009276354685425758,0.011895835399627686,0.010411957278847694,0.007498918566852808],[0.024409933015704155,0.011438388377428055,0.02003093995153904,0.005118591710925102,0.015081214718520641,0.012334463186562061,0.015452546067535877,0.008602438494563103,0.014702466316521168,0.020766181871294975,0.009192772209644318,0.005703327711671591],[0.017897022888064384,0.013280634768307209,0.0067551820538938046,0.012744802981615067,0.008020908571779728,0.007722198497503996,0.01734135113656521,0.0074547044932842255,0.007832478731870651,0.008252141997218132,0.013642732053995132,0.012807462364435196],[0.004923720378428698,0.007951012812554836,0.00794787798076868,0.004564068745821714,0.010363367386162281,0.009582011960446835,0.01028773095458746,0.008320694789290428,0.002570011653006077,0.012810957618057728,0.008063891902565956,0.0065582930110394955]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Duplicate Token Scores"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="8ebfd231-d4f6-431d-9f92-7bc6b6f70cd9" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("8ebfd231-d4f6-431d-9f92-7bc6b6f70cd9")) {                    Plotly.newPlot(                        "8ebfd231-d4f6-431d-9f92-7bc6b6f70cd9",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.003192325122654438,0.1323632299900055,0.005006915889680386,1.0427492270537186e-05,0.0013110269792377949,0.7034571170806885,0.004262051545083523,0.0001649669575272128,0.0024746304843574762,0.0008572910446673632,0.018891464918851852,0.008690940216183662],[0.0002916363882832229,0.00013781932648271322,0.0015036164550110698,0.005392488092184067,0.001858389819972217,0.009062976576387882,0.012414469383656979,0.00224055303260684,0.005135663319379091,0.005220625549554825,0.005546468310058117,0.029750702902674675],[0.0024816489312797785,0.009442185051739216,0.00034561342909000814,0.00025915156584233046,0.005211671814322472,0.0005709662800654769,0.0015209924895316362,0.006313098594546318,0.001560926903039217,0.00042158187716268003,0.0001535943301860243,0.005160389002412558],[0.6775667071342468,0.002840489149093628,0.0007841649930924177,0.004712563939392567,0.006322908215224743,0.00620671221986413,0.0005474375793710351,0.0003782894345931709,0.0020156530663371086,0.007952774874866009,0.003576836548745632,0.0026087891310453415],[0.008604029193520546,0.0070285736583173275,0.00759880430996418,0.0034427959471940994,0.016561321914196014,0.005979627370834351,0.0048697832971811295,0.0007624494028277695,0.006062139756977558,0.007536678109318018,0.012022883631289005,1.055450974014882e-12],[0.00950299110263586,0.008562120608985424,0.0041626025922596455,0.003008681582286954,0.006847443990409374,0.004358135629445314,0.007669272366911173,0.009584203362464905,0.007618824020028114,0.004328039940446615,0.04140293970704079,0.009761848486959934],[0.00445613032206893,0.008873283863067627,0.007405218668282032,0.006224926561117172,0.007319129537791014,0.005623883102089167,0.01734965480864048,0.005529423244297504,0.0029201731085777283,0.008636709302663803,0.006222232710570097,0.008358956314623356],[0.003699968568980694,0.041079357266426086,0.04148266464471817,0.009313643909990788,0.009097038768231869,0.008774377405643463,0.007298501208424568,0.023312222212553024,0.00884333811700344,0.00987985823303461,0.017598574981093407,0.006039811763912439],[0.008986336179077625,0.028667127713561058,0.00889119878411293,0.010114572942256927,0.009737403132021427,0.00761164166033268,0.009763195179402828,0.005155565217137337,0.009276354685425758,0.011895835399627686,0.010411957278847694,0.007498918566852808],[0.024409933015704155,0.011438388377428055,0.02003093995153904,0.005118591710925102,0.015081214718520641,0.012334463186562061,0.015452546067535877,0.008602438494563103,0.014702466316521168,0.020766181871294975,0.009192772209644318,0.005703327711671591],[0.017897022888064384,0.013280634768307209,0.0067551820538938046,0.012744802981615067,0.008020908571779728,0.007722198497503996,0.01734135113656521,0.0074547044932842255,0.007832478731870651,0.008252141997218132,0.013642732053995132,0.012807462364435196],[0.004923720378428698,0.007951012812554836,0.00794787798076868,0.004564068745821714,0.010363367386162281,0.009582011960446835,0.01028773095458746,0.008320694789290428,0.002570011653006077,0.012810957618057728,0.008063891902565956,0.0065582930110394955]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Duplicate Token Scores"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('fae2e193-59a0-4811-84ef-98e7b34809aa');
+var gd = document.getElementById('8ebfd231-d4f6-431d-9f92-7bc6b6f70cd9');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2146,9 +2146,9 @@ <h4>Implications<a class="headerlink" href="#Implications" title="Permalink to t
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="ecf35438-ee23-4f6c-85d0-db50b0b55227" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("ecf35438-ee23-4f6c-85d0-db50b0b55227")) {                    Plotly.newPlot(                        "ecf35438-ee23-4f6c-85d0-db50b0b55227",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.004035576246678829,3.85936327802483e-05,0.003946060314774513,1.7428237697458826e-07,5.9896454331465065e-05,4.0836926928022876e-05,0.003501761006191373,0.00024610935361124575,0.003167978022247553,0.003010402200743556,0.0020935393404215574,0.008525436744093895],[0.0005264790961518884,0.0001567041763337329,0.0015079419827088714,0.005595334805548191,0.0018401179695501924,0.003887568600475788,0.005349150858819485,0.0046491725370287895,0.005880181211978197,0.007283915765583515,0.005552181042730808,0.00012677432096097618],[0.0022015569265931845,0.008784865029156208,0.0021590511314570904,0.0010448142420500517,0.005142332520335913,0.002251654164865613,0.0008376692421734333,0.0063524022698402405,0.00261817779392004,0.0010309149511158466,0.00015219332999549806,0.005351166240870953],[0.007752286270260811,0.003091607242822647,0.00136294006370008,0.004341903142631054,0.011233086697757244,0.006535585504025221,0.0009068045765161514,0.0006078635924495757,0.0028195830527693033,0.005254121031612158,0.004195701330900192,0.0025541584473103285],[0.007342719938606024,0.004788315389305353,0.007458813022822142,0.0033073171507567167,0.007871860638260841,0.004219716414809227,0.004172018263489008,0.0005154716782271862,0.008124986663460732,0.006826961878687143,0.008085506968200207,3.761463363005646e-11],[0.43377670645713806,0.9306100010871887,0.006382290739566088,0.003473056945949793,0.005501003935933113,0.9255975484848022,0.005381471943110228,0.007857300341129303,0.008637800812721252,0.015764445066452026,0.012188390828669071,0.008265750482678413],[0.0025073012802749872,0.008432051166892052,0.008623287081718445,0.007653359789401293,0.011058016680181026,0.005525414831936359,0.017205143347382545,0.004794336389750242,0.004097685217857361,0.9257786273956299,0.020375533029437065,0.006313955411314964],[0.005555589683353901,0.18942901492118835,0.8509916663169861,0.008273174054920673,0.008239682763814926,0.00864996574819088,0.028328388929367065,0.08996234834194183,0.0066174231469631195,0.009413909167051315,0.9037811756134033,0.03037119098007679],[0.0073545570485293865,0.3791305124759674,0.005602711346000433,0.025401653721928596,0.008504705503582954,0.00623104115948081,0.1189238503575325,0.0051146638579666615,0.013350976631045341,0.01576736569404602,0.025844166055321693,0.008429795503616333],[0.23989056050777435,0.14378714561462402,0.09330623596906662,0.005819808691740036,0.07744759321212769,0.01644795574247837,0.4442824423313141,0.011141379363834858,0.03618974611163139,0.47264590859413147,0.008039995096623898,0.030952973291277885],[0.36065515875816345,0.4820112884044647,0.02285100519657135,0.12641923129558563,0.04125585779547691,0.007237402722239494,0.287715345621109,0.3897298574447632,0.0300607867538929,0.006112930364906788,0.16554805636405945,0.2224510908126831],[0.007408502046018839,0.033737070858478546,0.020412690937519073,0.0027554105035960674,0.02518630214035511,0.07808848470449448,0.033082809299230576,0.046439945697784424,0.0032543179113417864,0.2744251787662506,0.3800223767757416,0.009483428671956062]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Induction Head Scores"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="2926cb84-a4e5-40c5-bca4-0dcd0eb9b20f" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("2926cb84-a4e5-40c5-bca4-0dcd0eb9b20f")) {                    Plotly.newPlot(                        "2926cb84-a4e5-40c5-bca4-0dcd0eb9b20f",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.004035576246678829,3.85936327802483e-05,0.003946060314774513,1.7428237697458826e-07,5.9896454331465065e-05,4.0836926928022876e-05,0.003501761006191373,0.00024610935361124575,0.003167978022247553,0.003010402200743556,0.0020935393404215574,0.008525436744093895],[0.0005264790961518884,0.0001567041763337329,0.0015079419827088714,0.005595334805548191,0.0018401179695501924,0.003887568600475788,0.005349150858819485,0.0046491725370287895,0.005880181211978197,0.007283915765583515,0.005552181042730808,0.00012677432096097618],[0.0022015569265931845,0.008784865029156208,0.0021590511314570904,0.0010448142420500517,0.005142332520335913,0.002251654164865613,0.0008376692421734333,0.0063524022698402405,0.00261817779392004,0.0010309149511158466,0.00015219332999549806,0.005351166240870953],[0.007752286270260811,0.003091607242822647,0.00136294006370008,0.004341903142631054,0.011233086697757244,0.006535585504025221,0.0009068045765161514,0.0006078635924495757,0.0028195830527693033,0.005254121031612158,0.004195701330900192,0.0025541584473103285],[0.007342719938606024,0.004788315389305353,0.007458813022822142,0.0033073171507567167,0.007871860638260841,0.004219716414809227,0.004172018263489008,0.0005154716782271862,0.008124986663460732,0.006826961878687143,0.008085506968200207,3.761463363005646e-11],[0.43377670645713806,0.9306100010871887,0.006382290739566088,0.003473056945949793,0.005501003935933113,0.9255975484848022,0.005381471943110228,0.007857300341129303,0.008637800812721252,0.015764445066452026,0.012188390828669071,0.008265750482678413],[0.0025073012802749872,0.008432051166892052,0.008623287081718445,0.007653359789401293,0.011058016680181026,0.005525414831936359,0.017205143347382545,0.004794336389750242,0.004097685217857361,0.9257786273956299,0.020375533029437065,0.006313955411314964],[0.005555589683353901,0.18942901492118835,0.8509916663169861,0.008273174054920673,0.008239682763814926,0.00864996574819088,0.028328388929367065,0.08996234834194183,0.0066174231469631195,0.009413909167051315,0.9037811756134033,0.03037119098007679],[0.0073545570485293865,0.3791305124759674,0.005602711346000433,0.025401653721928596,0.008504705503582954,0.00623104115948081,0.1189238503575325,0.0051146638579666615,0.013350976631045341,0.01576736569404602,0.025844166055321693,0.008429795503616333],[0.23989056050777435,0.14378714561462402,0.09330623596906662,0.005819808691740036,0.07744759321212769,0.01644795574247837,0.4442824423313141,0.011141379363834858,0.03618974611163139,0.47264590859413147,0.008039995096623898,0.030952973291277885],[0.36065515875816345,0.4820112884044647,0.02285100519657135,0.12641923129558563,0.04125585779547691,0.007237402722239494,0.287715345621109,0.3897298574447632,0.0300607867538929,0.006112930364906788,0.16554805636405945,0.2224510908126831],[0.007408502046018839,0.033737070858478546,0.020412690937519073,0.0027554105035960674,0.02518630214035511,0.07808848470449448,0.033082809299230576,0.046439945697784424,0.0032543179113417864,0.2744251787662506,0.3800223767757416,0.009483428671956062]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Induction Head Scores"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('ecf35438-ee23-4f6c-85d0-db50b0b55227');
+var gd = document.getElementById('2926cb84-a4e5-40c5-bca4-0dcd0eb9b20f');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2278,9 +2278,9 @@ <h3>Backup Name Mover Heads<a class="headerlink" href="#Backup-Name-Mover-Heads"
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="2ed18f74-fde1-4c81-ae6d-442fa345fd56" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("2ed18f74-fde1-4c81-ae6d-442fa345fd56")) {                    Plotly.newPlot(                        "2ed18f74-fde1-4c81-ae6d-442fa345fd56",                        [{"coloraxis":"coloraxis","name":"0","z":[[-0.002156495349481702,-0.00046507216757163405,0.00024167195078916848,0.0002806607517413795,-0.0004162961558904499,-0.0004892406286671758,-0.0026209622155874968,-0.0029356717132031918,0.0004256087704561651,0.0005418135551735759,0.00023754549329169095,-7.4885378126055e-05],[-0.0006585237570106983,0.00040605897083878517,-0.0009330313769169152,0.0008937875390984118,-0.0009785089641809464,-0.0005338447517715394,-0.002798808738589287,-0.0042141154408454895,0.0025785760954022408,0.002450673608109355,0.0005351657164283097,0.0012349945027381182],[0.0009405009332112968,-0.0011168718338012695,-0.0011542299762368202,-0.001569736865349114,-0.0005699718603864312,0.0014514633221551776,0.0024399259127676487,0.003158304840326309,0.0009236858459189534,-0.0035781036131083965,-0.001065023592673242,-0.00035587642923928797],[-0.0005624899640679359,-1.1904543498530984e-05,0.0011530885240063071,0.0007360292365774512,0.0016493821749463677,0.0008800605428405106,-0.0006905477494001389,-0.0030319932848215103,0.0008080820553004742,0.00010364956688135862,-0.000580791849642992,-0.0011067147133871913],[-0.0026376410387456417,0.0002693022252060473,-0.0016417787410318851,-0.0034070289693772793,0.0017449733568355441,0.00046448662760667503,-0.0007899348856881261,0.0018328834557905793,-0.0008632306708022952,-0.00039788917638361454,0.0007879154873080552,-0.00012039285502396524],[0.0008688486414030194,0.0009473506361246109,-0.002280982444062829,-0.0011803142260760069,0.0002408194704912603,-0.0004318461287766695,-0.00037281878758221865,-0.0007385292556136847,0.0008114340016618371,-0.00040442857425659895,-0.007074241526424885,0.003946421667933464],[-0.014917591586709023,-0.0022801952436566353,0.0022679539397358894,-8.297240128740668e-05,-0.0049810330383479595,0.0027670827694237232,0.0062661729753017426,-0.0034861767198890448,-0.0013347702333703637,-0.001791873830370605,-0.0012231743894517422,0.0004053567536175251],[-0.0002461824333295226,-0.005790156312286854,-0.0004975919146090746,0.14218276739120483,-0.0014962118584662676,-0.01900637522339821,0.003133367281407118,-0.0018581977346912026,-0.011305071413516998,0.19225995242595673,-0.0011892454931512475,-0.0010282358853146434],[-0.0038004028610885143,-0.0008571104845032096,-0.013956702314317226,0.008289228193461895,0.004315000958740711,-0.009073707275092602,-0.08315176516771317,0.003456807229667902,-0.01805501990020275,0.00217801658436656,0.29780468344688416,0.024093760177493095],[0.08904657512903214,-0.0007931562722660601,0.07247605919837952,0.01501650083810091,-0.021209243685007095,0.05205458402633667,1.441115140914917,0.04743725806474686,-0.0322909839451313,0.0,0.0019993027672171593,-0.008072329685091972],[0.8600811958312988,0.32600724697113037,0.16344299912452698,0.07133562117815018,-0.004448350518941879,0.0006813746877014637,0.3661348223686218,-0.7105053663253784,-0.0020313761197030544,-0.03214346989989281,1.2294316291809082,0.0018451516516506672],[0.016877306625247,-0.0017300043255090714,-0.5010855793952942,0.02749773859977722,-0.005966317839920521,-0.004944117274135351,-0.08855222165584564,0.006622340530157089,0.04412432760000229,-0.02726682648062706,-1.1349124908447266,0.022879652678966522]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"margin":{"t":60}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="2d8ff8a7-fc76-4e41-9668-0433e68e918b" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("2d8ff8a7-fc76-4e41-9668-0433e68e918b")) {                    Plotly.newPlot(                        "2d8ff8a7-fc76-4e41-9668-0433e68e918b",                        [{"coloraxis":"coloraxis","name":"0","z":[[-0.002156495349481702,-0.00046507216757163405,0.00024167195078916848,0.0002806607517413795,-0.0004162961558904499,-0.0004892406286671758,-0.0026209622155874968,-0.0029356717132031918,0.0004256087704561651,0.0005418135551735759,0.00023754549329169095,-7.4885378126055e-05],[-0.0006585237570106983,0.00040605897083878517,-0.0009330313769169152,0.0008937875390984118,-0.0009785089641809464,-0.0005338447517715394,-0.002798808738589287,-0.0042141154408454895,0.0025785760954022408,0.002450673608109355,0.0005351657164283097,0.0012349945027381182],[0.0009405009332112968,-0.0011168718338012695,-0.0011542299762368202,-0.001569736865349114,-0.0005699718603864312,0.0014514633221551776,0.0024399259127676487,0.003158304840326309,0.0009236858459189534,-0.0035781036131083965,-0.001065023592673242,-0.00035587642923928797],[-0.0005624899640679359,-1.1904543498530984e-05,0.0011530885240063071,0.0007360292365774512,0.0016493821749463677,0.0008800605428405106,-0.0006905477494001389,-0.0030319932848215103,0.0008080820553004742,0.00010364956688135862,-0.000580791849642992,-0.0011067147133871913],[-0.0026376410387456417,0.0002693022252060473,-0.0016417787410318851,-0.0034070289693772793,0.0017449733568355441,0.00046448662760667503,-0.0007899348856881261,0.0018328834557905793,-0.0008632306708022952,-0.00039788917638361454,0.0007879154873080552,-0.00012039285502396524],[0.0008688486414030194,0.0009473506361246109,-0.002280982444062829,-0.0011803142260760069,0.0002408194704912603,-0.0004318461287766695,-0.00037281878758221865,-0.0007385292556136847,0.0008114340016618371,-0.00040442857425659895,-0.007074241526424885,0.003946421667933464],[-0.014917591586709023,-0.0022801952436566353,0.0022679539397358894,-8.297240128740668e-05,-0.0049810330383479595,0.0027670827694237232,0.0062661729753017426,-0.0034861767198890448,-0.0013347702333703637,-0.001791873830370605,-0.0012231743894517422,0.0004053567536175251],[-0.0002461824333295226,-0.005790156312286854,-0.0004975919146090746,0.14218276739120483,-0.0014962118584662676,-0.01900637522339821,0.003133367281407118,-0.0018581977346912026,-0.011305071413516998,0.19225995242595673,-0.0011892454931512475,-0.0010282358853146434],[-0.0038004028610885143,-0.0008571104845032096,-0.013956702314317226,0.008289228193461895,0.004315000958740711,-0.009073707275092602,-0.08315176516771317,0.003456807229667902,-0.01805501990020275,0.00217801658436656,0.29780468344688416,0.024093760177493095],[0.08904657512903214,-0.0007931562722660601,0.07247605919837952,0.01501650083810091,-0.021209243685007095,0.05205458402633667,1.441115140914917,0.04743725806474686,-0.0322909839451313,0.0,0.0019993027672171593,-0.008072329685091972],[0.8600811958312988,0.32600724697113037,0.16344299912452698,0.07133562117815018,-0.004448350518941879,0.0006813746877014637,0.3661348223686218,-0.7105053663253784,-0.0020313761197030544,-0.03214346989989281,1.2294316291809082,0.0018451516516506672],[0.016877306625247,-0.0017300043255090714,-0.5010855793952942,0.02749773859977722,-0.005966317839920521,-0.004944117274135351,-0.08855222165584564,0.006622340530157089,0.04412432760000229,-0.02726682648062706,-1.1349124908447266,0.022879652678966522]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"margin":{"t":60}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('2ed18f74-fde1-4c81-ae6d-442fa345fd56');
+var gd = document.getElementById('2d8ff8a7-fc76-4e41-9668-0433e68e918b');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2308,9 +2308,9 @@ <h3>Backup Name Mover Heads<a class="headerlink" href="#Backup-Name-Mover-Heads"
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="69a381a6-98d5-41d7-b432-3b083c9cf66c" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("69a381a6-98d5-41d7-b432-3b083c9cf66c")) {                    Plotly.newPlot(                        "69a381a6-98d5-41d7-b432-3b083c9cf66c",                        [{"hovertemplate":"\u003cb\u003e%{hovertext}\u003c\u002fb\u003e\u003cbr\u003e\u003cbr\u003eAblated=%{x}\u003cbr\u003eOriginal=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","hovertext":["L0H0","L0H1","L0H2","L0H3","L0H4","L0H5","L0H6","L0H7","L0H8","L0H9","L0H10","L0H11","L1H0","L1H1","L1H2","L1H3","L1H4","L1H5","L1H6","L1H7","L1H8","L1H9","L1H10","L1H11","L2H0","L2H1","L2H2","L2H3","L2H4","L2H5","L2H6","L2H7","L2H8","L2H9","L2H10","L2H11","L3H0","L3H1","L3H2","L3H3","L3H4","L3H5","L3H6","L3H7","L3H8","L3H9","L3H10","L3H11","L4H0","L4H1","L4H2","L4H3","L4H4","L4H5","L4H6","L4H7","L4H8","L4H9","L4H10","L4H11","L5H0","L5H1","L5H2","L5H3","L5H4","L5H5","L5H6","L5H7","L5H8","L5H9","L5H10","L5H11","L6H0","L6H1","L6H2","L6H3","L6H4","L6H5","L6H6","L6H7","L6H8","L6H9","L6H10","L6H11","L7H0","L7H1","L7H2","L7H3","L7H4","L7H5","L7H6","L7H7","L7H8","L7H9","L7H10","L7H11","L8H0","L8H1","L8H2","L8H3","L8H4","L8H5","L8H6","L8H7","L8H8","L8H9","L8H10","L8H11","L9H0","L9H1","L9H2","L9H3","L9H4","L9H5","L9H6","L9H7","L9H8","L9H9","L9H10","L9H11","L10H0","L10H1","L10H2","L10H3","L10H4","L10H5","L10H6","L10H7","L10H8","L10H9","L10H10","L10H11","L11H0","L11H1","L11H2","L11H3","L11H4","L11H5","L11H6","L11H7","L11H8","L11H9","L11H10","L11H11"],"legendgroup":"","marker":{"color":"#636efa","symbol":"circle"},"mode":"markers","name":"","orientation":"v","showlegend":false,"x":[-0.002156495349481702,-0.00046507216757163405,0.00024167195078916848,0.0002806607517413795,-0.0004162961558904499,-0.0004892406286671758,-0.0026209622155874968,-0.0029356717132031918,0.0004256087704561651,0.0005418135551735759,0.00023754549329169095,-7.4885378126055e-05,-0.0006585237570106983,0.00040605897083878517,-0.0009330313769169152,0.0008937875390984118,-0.0009785089641809464,-0.0005338447517715394,-0.002798808738589287,-0.0042141154408454895,0.0025785760954022408,0.002450673608109355,0.0005351657164283097,0.0012349945027381182,0.0009405009332112968,-0.0011168718338012695,-0.0011542299762368202,-0.001569736865349114,-0.0005699718603864312,0.0014514633221551776,0.0024399259127676487,0.003158304840326309,0.0009236858459189534,-0.0035781036131083965,-0.001065023592673242,-0.00035587642923928797,-0.0005624899640679359,-1.1904543498530984e-05,0.0011530885240063071,0.0007360292365774512,0.0016493821749463677,0.0008800605428405106,-0.0006905477494001389,-0.0030319932848215103,0.0008080820553004742,0.00010364956688135862,-0.000580791849642992,-0.0011067147133871913,-0.0026376410387456417,0.0002693022252060473,-0.0016417787410318851,-0.0034070289693772793,0.0017449733568355441,0.00046448662760667503,-0.0007899348856881261,0.0018328834557905793,-0.0008632306708022952,-0.00039788917638361454,0.0007879154873080552,-0.00012039285502396524,0.0008688486414030194,0.0009473506361246109,-0.002280982444062829,-0.0011803142260760069,0.0002408194704912603,-0.0004318461287766695,-0.00037281878758221865,-0.0007385292556136847,0.0008114340016618371,-0.00040442857425659895,-0.007074241526424885,0.003946421667933464,-0.014917591586709023,-0.0022801952436566353,0.0022679539397358894,-8.297240128740668e-05,-0.0049810330383479595,0.0027670827694237232,0.0062661729753017426,-0.0034861767198890448,-0.0013347702333703637,-0.001791873830370605,-0.0012231743894517422,0.0004053567536175251,-0.0002461824333295226,-0.005790156312286854,-0.0004975919146090746,0.14218276739120483,-0.0014962118584662676,-0.01900637522339821,0.003133367281407118,-0.0018581977346912026,-0.011305071413516998,0.19225995242595673,-0.0011892454931512475,-0.0010282358853146434,-0.0038004028610885143,-0.0008571104845032096,-0.013956702314317226,0.008289228193461895,0.004315000958740711,-0.009073707275092602,-0.08315176516771317,0.003456807229667902,-0.01805501990020275,0.00217801658436656,0.29780468344688416,0.024093760177493095,0.08904657512903214,-0.0007931562722660601,0.07247605919837952,0.01501650083810091,-0.021209243685007095,0.05205458402633667,1.441115140914917,0.04743725806474686,-0.0322909839451313,0.0,0.0019993027672171593,-0.008072329685091972,0.8600811958312988,0.32600724697113037,0.16344299912452698,0.07133562117815018,-0.004448350518941879,0.0006813746877014637,0.3661348223686218,-0.7105053663253784,-0.0020313761197030544,-0.03214346989989281,1.2294316291809082,0.0018451516516506672,0.016877306625247,-0.0017300043255090714,-0.5010855793952942,0.02749773859977722,-0.005966317839920521,-0.004944117274135351,-0.08855222165584564,0.006622340530157089,0.04412432760000229,-0.02726682648062706,-1.1349124908447266,0.022879652678966522],"xaxis":"x","y":[-0.002056329045444727,-0.0005102002760395408,0.0004685759777203202,0.00012511858949437737,-0.0006028746138326824,-0.00024295759794767946,-0.0023188991472125053,-0.0027583539485931396,0.000564592657610774,0.0009697366622276604,-0.0002504411095287651,4.7396752052009106e-06,-0.0010071106953546405,0.0003947066143155098,-0.001548723317682743,0.0014034844934940338,-0.0012652697041630745,-0.0011358254123479128,-0.0028159399516880512,-0.0029645331669598818,0.0029190238565206528,0.0025743518490344286,0.00036237656604498625,0.0017548904288560152,0.0005569161148741841,-0.0011266364017501473,-0.0017354193842038512,-0.0014514740323647857,-0.0002874041674658656,0.0017210585065186024,0.0026659027207642794,0.0031146786641329527,0.0005667298682965338,-0.003666497301310301,-0.0018847138853743672,7.028633262962103e-06,-0.0007264401647262275,0.0001136981591116637,0.0014301439514383674,0.0007490518037229776,0.002018478699028492,0.0007436765008606017,-0.0004617785452865064,-0.0039057875983417034,0.0011407355777919292,-4.027353134006262e-05,-0.0013293151278048754,-0.0017636835109442472,-0.0028281747363507748,0.0003364539588801563,-0.0014249025844037533,-0.003777292789891362,0.001599886454641819,0.00029889593133702874,-0.0008046383736655116,0.0020388178527355194,-0.001559373107738793,-0.0006437154370360076,0.001116809668019414,-0.0003500327584333718,0.0011338141048327088,0.0011259093880653381,-0.0025163700338453054,-0.0014790259301662445,0.00038791849510744214,-6.407558976206928e-05,-0.0005096771637909114,-0.0008841846138238907,0.0006399258272722363,-0.0010097046615555882,-0.006759032607078552,0.0033667325042188168,-0.015147397294640541,-0.0021350914612412453,0.0025931934360414743,-0.00042673421557992697,-0.005559004843235016,0.002665933221578598,0.0064108846709132195,-0.0038270088844001293,-0.0003842375008389354,-0.0016430213581770658,-0.001334429020062089,-9.183748625218868e-05,-9.488424984738231e-05,-0.005788922775536776,-0.0006383699947036803,0.134933739900589,-0.0017687629442662,-0.018917974084615707,0.0038733729161322117,-0.0021450738422572613,-0.010327240452170372,0.1832587718963623,-0.0007747872150503099,-0.0010452070273458958,-0.0038339472375810146,-0.0008046274306252599,-0.01267334446310997,0.008045826107263565,0.003604009747505188,-0.009398158639669418,-0.08272106945514679,0.0035550370812416077,-0.01840413361787796,0.001758674974553287,0.28961312770843506,0.022854033857584,0.08595199882984161,-0.000693259877152741,0.06816964596509933,0.01311142835766077,-0.02109798789024353,0.05112443491816521,1.384489893913269,0.045837316662073135,-0.038303449749946594,2.9854443073272705,0.001966138370335102,-0.008030208759009838,0.5608724355697632,0.17083144187927246,-0.033618729561567307,0.05821548402309418,-0.0024530175141990185,0.0018771894974634051,0.2882729470729828,-1.8986413478851318,-0.0015286938287317753,-0.035130057483911514,0.48021769523620605,-0.0009116912842728198,0.016075868159532547,-0.03986111283302307,-0.3879111707210541,0.01112320739775896,-0.0054778107441961765,-0.002512941136956215,-0.08056112378835678,0.007518645375967026,0.04301108419895172,-0.040082402527332306,-0.9702335000038147,0.011862380430102348],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"Ablated"},"range":[-3,3]},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"Original"},"range":[-3,3]},"legend":{"tracegroupgap":0},"title":{"text":"Original vs Post-Ablation Direct Logit Attribution of Heads"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="df924299-a4ee-4575-87e6-763165906024" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("df924299-a4ee-4575-87e6-763165906024")) {                    Plotly.newPlot(                        "df924299-a4ee-4575-87e6-763165906024",                        [{"hovertemplate":"\u003cb\u003e%{hovertext}\u003c\u002fb\u003e\u003cbr\u003e\u003cbr\u003eAblated=%{x}\u003cbr\u003eOriginal=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","hovertext":["L0H0","L0H1","L0H2","L0H3","L0H4","L0H5","L0H6","L0H7","L0H8","L0H9","L0H10","L0H11","L1H0","L1H1","L1H2","L1H3","L1H4","L1H5","L1H6","L1H7","L1H8","L1H9","L1H10","L1H11","L2H0","L2H1","L2H2","L2H3","L2H4","L2H5","L2H6","L2H7","L2H8","L2H9","L2H10","L2H11","L3H0","L3H1","L3H2","L3H3","L3H4","L3H5","L3H6","L3H7","L3H8","L3H9","L3H10","L3H11","L4H0","L4H1","L4H2","L4H3","L4H4","L4H5","L4H6","L4H7","L4H8","L4H9","L4H10","L4H11","L5H0","L5H1","L5H2","L5H3","L5H4","L5H5","L5H6","L5H7","L5H8","L5H9","L5H10","L5H11","L6H0","L6H1","L6H2","L6H3","L6H4","L6H5","L6H6","L6H7","L6H8","L6H9","L6H10","L6H11","L7H0","L7H1","L7H2","L7H3","L7H4","L7H5","L7H6","L7H7","L7H8","L7H9","L7H10","L7H11","L8H0","L8H1","L8H2","L8H3","L8H4","L8H5","L8H6","L8H7","L8H8","L8H9","L8H10","L8H11","L9H0","L9H1","L9H2","L9H3","L9H4","L9H5","L9H6","L9H7","L9H8","L9H9","L9H10","L9H11","L10H0","L10H1","L10H2","L10H3","L10H4","L10H5","L10H6","L10H7","L10H8","L10H9","L10H10","L10H11","L11H0","L11H1","L11H2","L11H3","L11H4","L11H5","L11H6","L11H7","L11H8","L11H9","L11H10","L11H11"],"legendgroup":"","marker":{"color":"#636efa","symbol":"circle"},"mode":"markers","name":"","orientation":"v","showlegend":false,"x":[-0.002156495349481702,-0.00046507216757163405,0.00024167195078916848,0.0002806607517413795,-0.0004162961558904499,-0.0004892406286671758,-0.0026209622155874968,-0.0029356717132031918,0.0004256087704561651,0.0005418135551735759,0.00023754549329169095,-7.4885378126055e-05,-0.0006585237570106983,0.00040605897083878517,-0.0009330313769169152,0.0008937875390984118,-0.0009785089641809464,-0.0005338447517715394,-0.002798808738589287,-0.0042141154408454895,0.0025785760954022408,0.002450673608109355,0.0005351657164283097,0.0012349945027381182,0.0009405009332112968,-0.0011168718338012695,-0.0011542299762368202,-0.001569736865349114,-0.0005699718603864312,0.0014514633221551776,0.0024399259127676487,0.003158304840326309,0.0009236858459189534,-0.0035781036131083965,-0.001065023592673242,-0.00035587642923928797,-0.0005624899640679359,-1.1904543498530984e-05,0.0011530885240063071,0.0007360292365774512,0.0016493821749463677,0.0008800605428405106,-0.0006905477494001389,-0.0030319932848215103,0.0008080820553004742,0.00010364956688135862,-0.000580791849642992,-0.0011067147133871913,-0.0026376410387456417,0.0002693022252060473,-0.0016417787410318851,-0.0034070289693772793,0.0017449733568355441,0.00046448662760667503,-0.0007899348856881261,0.0018328834557905793,-0.0008632306708022952,-0.00039788917638361454,0.0007879154873080552,-0.00012039285502396524,0.0008688486414030194,0.0009473506361246109,-0.002280982444062829,-0.0011803142260760069,0.0002408194704912603,-0.0004318461287766695,-0.00037281878758221865,-0.0007385292556136847,0.0008114340016618371,-0.00040442857425659895,-0.007074241526424885,0.003946421667933464,-0.014917591586709023,-0.0022801952436566353,0.0022679539397358894,-8.297240128740668e-05,-0.0049810330383479595,0.0027670827694237232,0.0062661729753017426,-0.0034861767198890448,-0.0013347702333703637,-0.001791873830370605,-0.0012231743894517422,0.0004053567536175251,-0.0002461824333295226,-0.005790156312286854,-0.0004975919146090746,0.14218276739120483,-0.0014962118584662676,-0.01900637522339821,0.003133367281407118,-0.0018581977346912026,-0.011305071413516998,0.19225995242595673,-0.0011892454931512475,-0.0010282358853146434,-0.0038004028610885143,-0.0008571104845032096,-0.013956702314317226,0.008289228193461895,0.004315000958740711,-0.009073707275092602,-0.08315176516771317,0.003456807229667902,-0.01805501990020275,0.00217801658436656,0.29780468344688416,0.024093760177493095,0.08904657512903214,-0.0007931562722660601,0.07247605919837952,0.01501650083810091,-0.021209243685007095,0.05205458402633667,1.441115140914917,0.04743725806474686,-0.0322909839451313,0.0,0.0019993027672171593,-0.008072329685091972,0.8600811958312988,0.32600724697113037,0.16344299912452698,0.07133562117815018,-0.004448350518941879,0.0006813746877014637,0.3661348223686218,-0.7105053663253784,-0.0020313761197030544,-0.03214346989989281,1.2294316291809082,0.0018451516516506672,0.016877306625247,-0.0017300043255090714,-0.5010855793952942,0.02749773859977722,-0.005966317839920521,-0.004944117274135351,-0.08855222165584564,0.006622340530157089,0.04412432760000229,-0.02726682648062706,-1.1349124908447266,0.022879652678966522],"xaxis":"x","y":[-0.002056329045444727,-0.0005102002760395408,0.0004685759777203202,0.00012511858949437737,-0.0006028746138326824,-0.00024295759794767946,-0.0023188991472125053,-0.0027583539485931396,0.000564592657610774,0.0009697366622276604,-0.0002504411095287651,4.7396752052009106e-06,-0.0010071106953546405,0.0003947066143155098,-0.001548723317682743,0.0014034844934940338,-0.0012652697041630745,-0.0011358254123479128,-0.0028159399516880512,-0.0029645331669598818,0.0029190238565206528,0.0025743518490344286,0.00036237656604498625,0.0017548904288560152,0.0005569161148741841,-0.0011266364017501473,-0.0017354193842038512,-0.0014514740323647857,-0.0002874041674658656,0.0017210585065186024,0.0026659027207642794,0.0031146786641329527,0.0005667298682965338,-0.003666497301310301,-0.0018847138853743672,7.028633262962103e-06,-0.0007264401647262275,0.0001136981591116637,0.0014301439514383674,0.0007490518037229776,0.002018478699028492,0.0007436765008606017,-0.0004617785452865064,-0.0039057875983417034,0.0011407355777919292,-4.027353134006262e-05,-0.0013293151278048754,-0.0017636835109442472,-0.0028281747363507748,0.0003364539588801563,-0.0014249025844037533,-0.003777292789891362,0.001599886454641819,0.00029889593133702874,-0.0008046383736655116,0.0020388178527355194,-0.001559373107738793,-0.0006437154370360076,0.001116809668019414,-0.0003500327584333718,0.0011338141048327088,0.0011259093880653381,-0.0025163700338453054,-0.0014790259301662445,0.00038791849510744214,-6.407558976206928e-05,-0.0005096771637909114,-0.0008841846138238907,0.0006399258272722363,-0.0010097046615555882,-0.006759032607078552,0.0033667325042188168,-0.015147397294640541,-0.0021350914612412453,0.0025931934360414743,-0.00042673421557992697,-0.005559004843235016,0.002665933221578598,0.0064108846709132195,-0.0038270088844001293,-0.0003842375008389354,-0.0016430213581770658,-0.001334429020062089,-9.183748625218868e-05,-9.488424984738231e-05,-0.005788922775536776,-0.0006383699947036803,0.134933739900589,-0.0017687629442662,-0.018917974084615707,0.0038733729161322117,-0.0021450738422572613,-0.010327240452170372,0.1832587718963623,-0.0007747872150503099,-0.0010452070273458958,-0.0038339472375810146,-0.0008046274306252599,-0.01267334446310997,0.008045826107263565,0.003604009747505188,-0.009398158639669418,-0.08272106945514679,0.0035550370812416077,-0.01840413361787796,0.001758674974553287,0.28961312770843506,0.022854033857584,0.08595199882984161,-0.000693259877152741,0.06816964596509933,0.01311142835766077,-0.02109798789024353,0.05112443491816521,1.384489893913269,0.045837316662073135,-0.038303449749946594,2.9854443073272705,0.001966138370335102,-0.008030208759009838,0.5608724355697632,0.17083144187927246,-0.033618729561567307,0.05821548402309418,-0.0024530175141990185,0.0018771894974634051,0.2882729470729828,-1.8986413478851318,-0.0015286938287317753,-0.035130057483911514,0.48021769523620605,-0.0009116912842728198,0.016075868159532547,-0.03986111283302307,-0.3879111707210541,0.01112320739775896,-0.0054778107441961765,-0.002512941136956215,-0.08056112378835678,0.007518645375967026,0.04301108419895172,-0.040082402527332306,-0.9702335000038147,0.011862380430102348],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"Ablated"},"range":[-3,3]},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"Original"},"range":[-3,3]},"legend":{"tracegroupgap":0},"title":{"text":"Original vs Post-Ablation Direct Logit Attribution of Heads"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('69a381a6-98d5-41d7-b432-3b083c9cf66c');
+var gd = document.getElementById('df924299-a4ee-4575-87e6-763165906024');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2388,7 +2388,7 @@ <h3>Backup Name Mover Heads<a class="headerlink" href="#Backup-Name-Mover-Heads"
 </div>
 <p><strong>Exercise to the reader:</strong> Can you finish off this analysis? What’s going on here? Why are the backup name movers changing their behaviour? Why is one negative name mover becoming significantly less important?</p>
 <script type="application/vnd.jupyter.widget-state+json">
-{"state": {"fe14fba431ee44719e467a94d38602c1": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "71480013d795482ea7827866fd7d1fb0": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "2a992a646fbc405db6ec21ba17538e0e": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_fe14fba431ee44719e467a94d38602c1", "max": 548105171.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_71480013d795482ea7827866fd7d1fb0", "tabbable": null, "tooltip": null, "value": 548105171.0}}, "b60c8dc4072a47f69bc60e5115ab87c6": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "3dd3706dad6149a2ae71bfb8a4a5256a": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "0ccd5c3ba66d4687bda0b8a38511eecd": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_b60c8dc4072a47f69bc60e5115ab87c6", "placeholder": "\u200b", "style": "IPY_MODEL_3dd3706dad6149a2ae71bfb8a4a5256a", "tabbable": null, "tooltip": null, "value": "model.safetensors:\u2007100%"}}, "1dffcf97acf041dfa4829930fc637e56": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "bf988798a38d454fa0c118280dbeab56": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "ddeeebc813c54775ab8841a88e512534": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_1dffcf97acf041dfa4829930fc637e56", "placeholder": "\u200b", "style": "IPY_MODEL_bf988798a38d454fa0c118280dbeab56", "tabbable": null, "tooltip": null, "value": "\u2007548M/548M\u2007[00:02&lt;00:00,\u2007243MB/s]"}}, "1acc38b256394079bf479fc9c3f6bbff": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "2566b58370ab4ee1a23ec40b298e524d": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_0ccd5c3ba66d4687bda0b8a38511eecd", "IPY_MODEL_2a992a646fbc405db6ec21ba17538e0e", "IPY_MODEL_ddeeebc813c54775ab8841a88e512534"], "layout": "IPY_MODEL_1acc38b256394079bf479fc9c3f6bbff", "tabbable": null, "tooltip": null}}, "f06d8ed2f2fc40c78e6f71a980042fa2": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "0947963163124b84ad7bdbd0d14a5852": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "53f61b927c1346d88f4d1f2962f875ba": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_f06d8ed2f2fc40c78e6f71a980042fa2", "max": 124.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_0947963163124b84ad7bdbd0d14a5852", "tabbable": null, "tooltip": null, "value": 124.0}}, "481fa4fd501940cebdf3f44be6c9e882": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "ed04372d5ce8433d848d96e975723537": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "53bd3bcec1c747a0a2f91fd51bd8fba0": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_481fa4fd501940cebdf3f44be6c9e882", "placeholder": "\u200b", "style": "IPY_MODEL_ed04372d5ce8433d848d96e975723537", "tabbable": null, "tooltip": null, "value": "generation_config.json:\u2007100%"}}, "4a23001ae4824747ac51dbe487a0a126": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "add5c8f3c40642ef8e4cf96579325ec3": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "c43b6d1b7dc04e8ea921c5940f890126": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_4a23001ae4824747ac51dbe487a0a126", "placeholder": "\u200b", "style": "IPY_MODEL_add5c8f3c40642ef8e4cf96579325ec3", "tabbable": null, "tooltip": null, "value": "\u2007124/124\u2007[00:00&lt;00:00,\u200724.5kB/s]"}}, "b1a0d471042a4a018d4de3bf29a3ecd2": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "4c4a1ec2b7514c1297506faae67c1374": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_53bd3bcec1c747a0a2f91fd51bd8fba0", "IPY_MODEL_53f61b927c1346d88f4d1f2962f875ba", "IPY_MODEL_c43b6d1b7dc04e8ea921c5940f890126"], "layout": "IPY_MODEL_b1a0d471042a4a018d4de3bf29a3ecd2", "tabbable": null, "tooltip": null}}, "853700c97ecf475a8336c57945c87c6a": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "47513102e87a47069219c229c4725eea": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "ba3727d9eb8e45b394738da6078d874c": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_853700c97ecf475a8336c57945c87c6a", "max": 26.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_47513102e87a47069219c229c4725eea", "tabbable": null, "tooltip": null, "value": 26.0}}, "3250ac219bb14a0d87380c723d7187c5": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "f6e7f731edb44410a1bbe854b20b75b3": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "7daabc68020641e897d4809176829307": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_3250ac219bb14a0d87380c723d7187c5", "placeholder": "\u200b", "style": "IPY_MODEL_f6e7f731edb44410a1bbe854b20b75b3", "tabbable": null, "tooltip": null, "value": "tokenizer_config.json:\u2007100%"}}, "c0e90bf5be684b4eadbd6f97b968ab31": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "e49babdcc96f4f99af8dba7cd03202fc": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "ac60beb4421b4e758aec659715fa5146": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_c0e90bf5be684b4eadbd6f97b968ab31", "placeholder": "\u200b", "style": "IPY_MODEL_e49babdcc96f4f99af8dba7cd03202fc", "tabbable": null, "tooltip": null, "value": "\u200726.0/26.0\u2007[00:00&lt;00:00,\u20074.60kB/s]"}}, "97133900a1a94bc691ce291364fca64c": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "54e358754fd241a7beb89dbd18d54469": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_7daabc68020641e897d4809176829307", "IPY_MODEL_ba3727d9eb8e45b394738da6078d874c", "IPY_MODEL_ac60beb4421b4e758aec659715fa5146"], "layout": "IPY_MODEL_97133900a1a94bc691ce291364fca64c", "tabbable": null, "tooltip": null}}, "488e5f543a8f48a79b17dd90afc28284": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "7ffcda766f894799bb645944b5ecab6b": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "6b5428d552484be2b498aa06f2cd12d7": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_488e5f543a8f48a79b17dd90afc28284", "max": 1042301.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_7ffcda766f894799bb645944b5ecab6b", "tabbable": null, "tooltip": null, "value": 1042301.0}}, "9351dee6b05946a0a6880580b6f6ff6b": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "924d96b2376c4b16a4541d64b0e45872": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "b9ffe90723d54a178c02c4a71079ef4f": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_9351dee6b05946a0a6880580b6f6ff6b", "placeholder": "\u200b", "style": "IPY_MODEL_924d96b2376c4b16a4541d64b0e45872", "tabbable": null, "tooltip": null, "value": "vocab.json:\u2007100%"}}, "4647f9260ab248d7a350e7a07c999a9b": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "7f602aa1ed364df1a417f0cf429e4a5c": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "d8504344e45a40dba0505fafd0a4fef8": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_4647f9260ab248d7a350e7a07c999a9b", "placeholder": "\u200b", "style": "IPY_MODEL_7f602aa1ed364df1a417f0cf429e4a5c", "tabbable": null, "tooltip": null, "value": "\u20071.04M/1.04M\u2007[00:00&lt;00:00,\u200725.8MB/s]"}}, "c655320743a4421684aca235370c46c1": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "c9ac7e38eb534678896df289998d1855": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_b9ffe90723d54a178c02c4a71079ef4f", "IPY_MODEL_6b5428d552484be2b498aa06f2cd12d7", "IPY_MODEL_d8504344e45a40dba0505fafd0a4fef8"], "layout": "IPY_MODEL_c655320743a4421684aca235370c46c1", "tabbable": null, "tooltip": null}}, "4faadec4ac0f4a4ab32961fb3b74e529": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "6ab9eaf5ee3342379119468f880a7e68": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "9a499b42e58849dcbfc030ebb0630a62": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_4faadec4ac0f4a4ab32961fb3b74e529", "max": 456318.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_6ab9eaf5ee3342379119468f880a7e68", "tabbable": null, "tooltip": null, "value": 456318.0}}, "621583ed869b478d913fc04a87073a95": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "d2847e7beeb442018670534e35889459": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "d3bf56862e814e46a08ce4949ce5df67": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_621583ed869b478d913fc04a87073a95", "placeholder": "\u200b", "style": "IPY_MODEL_d2847e7beeb442018670534e35889459", "tabbable": null, "tooltip": null, "value": "merges.txt:\u2007100%"}}, "de2a35b1b3264fd2ac593af14da5b645": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "c02c62e902e648ef9e245988ad7917f6": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "b4faa038d86840faa63b4ff7f8ac404c": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_de2a35b1b3264fd2ac593af14da5b645", "placeholder": "\u200b", "style": "IPY_MODEL_c02c62e902e648ef9e245988ad7917f6", "tabbable": null, "tooltip": null, "value": "\u2007456k/456k\u2007[00:00&lt;00:00,\u200738.7MB/s]"}}, "5fe7263a54b34f2b918ca2c57d194d74": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "0b727299d6b94848ba14d79205e0d847": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_d3bf56862e814e46a08ce4949ce5df67", "IPY_MODEL_9a499b42e58849dcbfc030ebb0630a62", "IPY_MODEL_b4faa038d86840faa63b4ff7f8ac404c"], "layout": "IPY_MODEL_5fe7263a54b34f2b918ca2c57d194d74", "tabbable": null, "tooltip": null}}, "860eac54afd644cdb66576f10d1c4b94": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "b2dd202e54cc4c4bbf18779cef088e63": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "c9d427e21ae84197b7ff8cdb8bb4c89d": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_860eac54afd644cdb66576f10d1c4b94", "max": 1355256.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_b2dd202e54cc4c4bbf18779cef088e63", "tabbable": null, "tooltip": null, "value": 1355256.0}}, "4973dab052f14376aa999fd6bc13197b": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "4064ed6431a14c8d940329269c2399e3": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "aa33f2b3f7af4a5c956ebdbae08dabec": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_4973dab052f14376aa999fd6bc13197b", "placeholder": "\u200b", "style": "IPY_MODEL_4064ed6431a14c8d940329269c2399e3", "tabbable": null, "tooltip": null, "value": "tokenizer.json:\u2007100%"}}, "910235bb167a412ab2a54c930635b3bf": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "f988b1609264444ea2e348b491a028a4": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "0dc2fdfccac3440ea6a358b4b727da08": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_910235bb167a412ab2a54c930635b3bf", "placeholder": "\u200b", "style": "IPY_MODEL_f988b1609264444ea2e348b491a028a4", "tabbable": null, "tooltip": null, "value": "\u20071.36M/1.36M\u2007[00:00&lt;00:00,\u200747.4MB/s]"}}, "bec18b05c0a8413485c9f6fb6d8d51dd": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "cf0868984e46408fb4a1d566ec3db0b0": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_aa33f2b3f7af4a5c956ebdbae08dabec", "IPY_MODEL_c9d427e21ae84197b7ff8cdb8bb4c89d", "IPY_MODEL_0dc2fdfccac3440ea6a358b4b727da08"], "layout": "IPY_MODEL_bec18b05c0a8413485c9f6fb6d8d51dd", "tabbable": null, "tooltip": null}}}, "version_major": 2, "version_minor": 0}
+{"state": {"8a9ecc907dfa4a7b97bcd057b4f96110": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "063d9fb16c374bbcbdee214535e4af62": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "8f5ab92f87714abab2c7cce99a7ef76d": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_8a9ecc907dfa4a7b97bcd057b4f96110", "max": 548105171.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_063d9fb16c374bbcbdee214535e4af62", "tabbable": null, "tooltip": null, "value": 548105171.0}}, "be0da02273224522ab03fcdf121a9d42": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "a3a152183bb84d98b4545d9f13d5980b": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "48e6660d282a4e2db01422526059d280": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_be0da02273224522ab03fcdf121a9d42", "placeholder": "\u200b", "style": "IPY_MODEL_a3a152183bb84d98b4545d9f13d5980b", "tabbable": null, "tooltip": null, "value": "model.safetensors:\u2007100%"}}, "75f267512b70424083e6f6741d3132e0": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "a8bd4932361949f2be691da2b7c4e787": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "896b626d27ed4333a78c8bc4b08fef82": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_75f267512b70424083e6f6741d3132e0", "placeholder": "\u200b", "style": "IPY_MODEL_a8bd4932361949f2be691da2b7c4e787", "tabbable": null, "tooltip": null, "value": "\u2007548M/548M\u2007[00:02&lt;00:00,\u2007201MB/s]"}}, "41f933e12f1d4d79a42601d7b7c9da82": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "e3f6e035556c4a02a0f9ccad87c2f138": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_48e6660d282a4e2db01422526059d280", "IPY_MODEL_8f5ab92f87714abab2c7cce99a7ef76d", "IPY_MODEL_896b626d27ed4333a78c8bc4b08fef82"], "layout": "IPY_MODEL_41f933e12f1d4d79a42601d7b7c9da82", "tabbable": null, "tooltip": null}}, "0f16d4bd6ea64fa6b925ab2efd1ca15e": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "449107685d8c439e8aaf9931f3acf205": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "b2196b99196b44458834657ac7229487": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_0f16d4bd6ea64fa6b925ab2efd1ca15e", "max": 124.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_449107685d8c439e8aaf9931f3acf205", "tabbable": null, "tooltip": null, "value": 124.0}}, "f2554ae1c6064ee7aaa80c39701f6c74": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "816c22a73aab42ab955422e98bcd057a": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "8f8c377a736d4e568f9f92ca89d5037d": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_f2554ae1c6064ee7aaa80c39701f6c74", "placeholder": "\u200b", "style": "IPY_MODEL_816c22a73aab42ab955422e98bcd057a", "tabbable": null, "tooltip": null, "value": "generation_config.json:\u2007100%"}}, "058fdb1f616145a989119197863c8fa2": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "175ab33428e24054a9588b575b136309": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "875219f2eae645a78d58982a80ba43d2": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_058fdb1f616145a989119197863c8fa2", "placeholder": "\u200b", "style": "IPY_MODEL_175ab33428e24054a9588b575b136309", "tabbable": null, "tooltip": null, "value": "\u2007124/124\u2007[00:00&lt;00:00,\u200721.4kB/s]"}}, "ae786b718eb746bf98a72ad65faeaa17": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "7fd3dba640c2447299e8235086a74547": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_8f8c377a736d4e568f9f92ca89d5037d", "IPY_MODEL_b2196b99196b44458834657ac7229487", "IPY_MODEL_875219f2eae645a78d58982a80ba43d2"], "layout": "IPY_MODEL_ae786b718eb746bf98a72ad65faeaa17", "tabbable": null, "tooltip": null}}, "44cc2a714dab4fbd92b45ec21a01bcef": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "91ad41831c3a4e5cadc2058781d8fb0d": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "aa6a0d5ee722417d9d0a0c721f6a32bf": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_44cc2a714dab4fbd92b45ec21a01bcef", "max": 26.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_91ad41831c3a4e5cadc2058781d8fb0d", "tabbable": null, "tooltip": null, "value": 26.0}}, "baa17a1c01be427c8dd2fe2e9a46b3fc": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "1257c8ebaf854ceba5d1ffbad2bc4d64": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "2bf41dd86aaf412b8d50d2cf2ab37cad": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_baa17a1c01be427c8dd2fe2e9a46b3fc", "placeholder": "\u200b", "style": "IPY_MODEL_1257c8ebaf854ceba5d1ffbad2bc4d64", "tabbable": null, "tooltip": null, "value": "tokenizer_config.json:\u2007100%"}}, "f94c5abf57df4cdba73ed4b8c8bafb54": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "35b095005ad54340aeff4c74af9f6f76": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "2dae86562a8d47b5a5c7935ebcdfea75": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_f94c5abf57df4cdba73ed4b8c8bafb54", "placeholder": "\u200b", "style": "IPY_MODEL_35b095005ad54340aeff4c74af9f6f76", "tabbable": null, "tooltip": null, "value": "\u200726.0/26.0\u2007[00:00&lt;00:00,\u20074.68kB/s]"}}, "d4bda90d6e2843c98dc87c59da4c233e": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "7473a57e0b514125b34860321843604d": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_2bf41dd86aaf412b8d50d2cf2ab37cad", "IPY_MODEL_aa6a0d5ee722417d9d0a0c721f6a32bf", "IPY_MODEL_2dae86562a8d47b5a5c7935ebcdfea75"], "layout": "IPY_MODEL_d4bda90d6e2843c98dc87c59da4c233e", "tabbable": null, "tooltip": null}}, "00869a4f30534799a76cd92f9815d895": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "0a2dc7a44b50480498bd6e0a8258b228": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "9cf230ec0aaf4f5590a6afd2c086e8a7": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_00869a4f30534799a76cd92f9815d895", "max": 1042301.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_0a2dc7a44b50480498bd6e0a8258b228", "tabbable": null, "tooltip": null, "value": 1042301.0}}, "32a2e286dd0e4b47a7ed286e57e25a02": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "19befccdb76441798bcd9c34f9a3970e": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "cb4cbc4865294ceca967b149417812dd": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_32a2e286dd0e4b47a7ed286e57e25a02", "placeholder": "\u200b", "style": "IPY_MODEL_19befccdb76441798bcd9c34f9a3970e", "tabbable": null, "tooltip": null, "value": "vocab.json:\u2007100%"}}, "f9181f78c4654800ada77429eb78dc13": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "220d51ccdafb40df864859c57034a447": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "a4de44337e914415b93595097425f7ac": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_f9181f78c4654800ada77429eb78dc13", "placeholder": "\u200b", "style": "IPY_MODEL_220d51ccdafb40df864859c57034a447", "tabbable": null, "tooltip": null, "value": "\u20071.04M/1.04M\u2007[00:00&lt;00:00,\u20075.31MB/s]"}}, "c47c55e36b944e7f9945ca4e513889f8": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "5af480c7ada14eb98937ccc79bb5be16": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_cb4cbc4865294ceca967b149417812dd", "IPY_MODEL_9cf230ec0aaf4f5590a6afd2c086e8a7", "IPY_MODEL_a4de44337e914415b93595097425f7ac"], "layout": "IPY_MODEL_c47c55e36b944e7f9945ca4e513889f8", "tabbable": null, "tooltip": null}}, "f2817ec73aab434a87ea26dff6fcd560": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "954188958f204f53bc21c9394f7c060c": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "f3b0ff1fd79f4f90800ffb7c4391e975": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_f2817ec73aab434a87ea26dff6fcd560", "max": 456318.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_954188958f204f53bc21c9394f7c060c", "tabbable": null, "tooltip": null, "value": 456318.0}}, "46075fcab25949e4accbc84c39c7b694": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "dafd098f48e24be49b4261213ec00625": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "7d793895e94448fabb3f09887d5be895": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_46075fcab25949e4accbc84c39c7b694", "placeholder": "\u200b", "style": "IPY_MODEL_dafd098f48e24be49b4261213ec00625", "tabbable": null, "tooltip": null, "value": "merges.txt:\u2007100%"}}, "1fd6a946b74140c8b91d0cfcc9c8a048": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "c4c48775aaab4649a5dc20fc1b976758": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "9bc7a1f74e244aeda156fded883d3fbc": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_1fd6a946b74140c8b91d0cfcc9c8a048", "placeholder": "\u200b", "style": "IPY_MODEL_c4c48775aaab4649a5dc20fc1b976758", "tabbable": null, "tooltip": null, "value": "\u2007456k/456k\u2007[00:00&lt;00:00,\u20073.52MB/s]"}}, "f6926ea8116f41a487c8a794a0461d52": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "fb98f7a2082142148838a198ceed048b": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_7d793895e94448fabb3f09887d5be895", "IPY_MODEL_f3b0ff1fd79f4f90800ffb7c4391e975", "IPY_MODEL_9bc7a1f74e244aeda156fded883d3fbc"], "layout": "IPY_MODEL_f6926ea8116f41a487c8a794a0461d52", "tabbable": null, "tooltip": null}}, "389fb2476c1e46eca84fb5e7bb0eec35": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "708a6bc613444a518f2b42ae989a958a": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "239292d605c54be48e65ecf57c0cb3cf": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_389fb2476c1e46eca84fb5e7bb0eec35", "max": 1355256.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_708a6bc613444a518f2b42ae989a958a", "tabbable": null, "tooltip": null, "value": 1355256.0}}, "da86d7929cf44490b4235842717812d2": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "46e2ce762f2046b0acf9a0538b70d087": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "2536618b59144bf68c23a2f9688d5ddb": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_da86d7929cf44490b4235842717812d2", "placeholder": "\u200b", "style": "IPY_MODEL_46e2ce762f2046b0acf9a0538b70d087", "tabbable": null, "tooltip": null, "value": "tokenizer.json:\u2007100%"}}, "6bbb6e62a049423bb1bb4a903cd0f7ef": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "4cc4e90000854566806a0f904f8413a5": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "28ddfba82ab54559ba0701ee4fa3195b": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_6bbb6e62a049423bb1bb4a903cd0f7ef", "placeholder": "\u200b", "style": "IPY_MODEL_4cc4e90000854566806a0f904f8413a5", "tabbable": null, "tooltip": null, "value": "\u20071.36M/1.36M\u2007[00:00&lt;00:00,\u20075.12MB/s]"}}, "5e99fedf15684824abf596d6692378e1": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "12f2342a4d7e48c9aaec6b2665da59ed": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_2536618b59144bf68c23a2f9688d5ddb", "IPY_MODEL_239292d605c54be48e65ecf57c0cb3cf", "IPY_MODEL_28ddfba82ab54559ba0701ee4fa3195b"], "layout": "IPY_MODEL_5e99fedf15684824abf596d6692378e1", "tabbable": null, "tooltip": null}}}, "version_major": 2, "version_minor": 0}
 </script></section>
 </section>
 </section>
diff --git a/generated/demos/Exploratory_Analysis_Demo.ipynb b/generated/demos/Exploratory_Analysis_Demo.ipynb
index 8dbd19640..b330c8c53 100644
--- a/generated/demos/Exploratory_Analysis_Demo.ipynb
+++ b/generated/demos/Exploratory_Analysis_Demo.ipynb
@@ -65,10 +65,10 @@
    "execution_count": 1,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T00:56:47.879779Z",
-     "iopub.status.busy": "2024-12-14T00:56:47.879335Z",
-     "iopub.status.idle": "2024-12-14T00:56:47.889435Z",
-     "shell.execute_reply": "2024-12-14T00:56:47.888895Z"
+     "iopub.execute_input": "2024-12-31T02:16:09.391631Z",
+     "iopub.status.busy": "2024-12-31T02:16:09.391147Z",
+     "iopub.status.idle": "2024-12-31T02:16:09.401395Z",
+     "shell.execute_reply": "2024-12-31T02:16:09.400946Z"
     }
    },
    "outputs": [],
@@ -110,10 +110,10 @@
    "execution_count": 2,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T00:56:47.891751Z",
-     "iopub.status.busy": "2024-12-14T00:56:47.891556Z",
-     "iopub.status.idle": "2024-12-14T00:56:51.109946Z",
-     "shell.execute_reply": "2024-12-14T00:56:51.109454Z"
+     "iopub.execute_input": "2024-12-31T02:16:09.403482Z",
+     "iopub.status.busy": "2024-12-31T02:16:09.403142Z",
+     "iopub.status.idle": "2024-12-31T02:16:12.576075Z",
+     "shell.execute_reply": "2024-12-31T02:16:12.575514Z"
     }
    },
    "outputs": [],
@@ -154,10 +154,10 @@
    "execution_count": 3,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T00:56:51.112619Z",
-     "iopub.status.busy": "2024-12-14T00:56:51.112256Z",
-     "iopub.status.idle": "2024-12-14T00:56:51.115224Z",
-     "shell.execute_reply": "2024-12-14T00:56:51.114674Z"
+     "iopub.execute_input": "2024-12-31T02:16:12.578860Z",
+     "iopub.status.busy": "2024-12-31T02:16:12.578493Z",
+     "iopub.status.idle": "2024-12-31T02:16:12.581750Z",
+     "shell.execute_reply": "2024-12-31T02:16:12.581195Z"
     }
    },
    "outputs": [
@@ -193,10 +193,10 @@
    "execution_count": 4,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T00:56:51.149823Z",
-     "iopub.status.busy": "2024-12-14T00:56:51.149327Z",
-     "iopub.status.idle": "2024-12-14T00:56:51.153801Z",
-     "shell.execute_reply": "2024-12-14T00:56:51.153349Z"
+     "iopub.execute_input": "2024-12-31T02:16:12.617218Z",
+     "iopub.status.busy": "2024-12-31T02:16:12.616754Z",
+     "iopub.status.idle": "2024-12-31T02:16:12.621311Z",
+     "shell.execute_reply": "2024-12-31T02:16:12.620819Z"
     }
    },
    "outputs": [],
@@ -281,17 +281,17 @@
    "execution_count": 5,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T00:56:51.155889Z",
-     "iopub.status.busy": "2024-12-14T00:56:51.155539Z",
-     "iopub.status.idle": "2024-12-14T00:56:55.978791Z",
-     "shell.execute_reply": "2024-12-14T00:56:55.978209Z"
+     "iopub.execute_input": "2024-12-31T02:16:12.623398Z",
+     "iopub.status.busy": "2024-12-31T02:16:12.623219Z",
+     "iopub.status.idle": "2024-12-31T02:16:19.621520Z",
+     "shell.execute_reply": "2024-12-31T02:16:19.620891Z"
     }
    },
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "2566b58370ab4ee1a23ec40b298e524d",
+       "model_id": "e3f6e035556c4a02a0f9ccad87c2f138",
        "version_major": 2,
        "version_minor": 0
       },
@@ -305,7 +305,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "4c4a1ec2b7514c1297506faae67c1374",
+       "model_id": "7fd3dba640c2447299e8235086a74547",
        "version_major": 2,
        "version_minor": 0
       },
@@ -319,7 +319,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "54e358754fd241a7beb89dbd18d54469",
+       "model_id": "7473a57e0b514125b34860321843604d",
        "version_major": 2,
        "version_minor": 0
       },
@@ -333,7 +333,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "c9ac7e38eb534678896df289998d1855",
+       "model_id": "5af480c7ada14eb98937ccc79bb5be16",
        "version_major": 2,
        "version_minor": 0
       },
@@ -347,7 +347,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "0b727299d6b94848ba14d79205e0d847",
+       "model_id": "fb98f7a2082142148838a198ceed048b",
        "version_major": 2,
        "version_minor": 0
       },
@@ -361,7 +361,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "cf0868984e46408fb4a1d566ec3db0b0",
+       "model_id": "12f2342a4d7e48c9aaec6b2665da59ed",
        "version_major": 2,
        "version_minor": 0
       },
@@ -413,10 +413,10 @@
    "execution_count": 6,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T00:56:55.981331Z",
-     "iopub.status.busy": "2024-12-14T00:56:55.980946Z",
-     "iopub.status.idle": "2024-12-14T00:56:56.260772Z",
-     "shell.execute_reply": "2024-12-14T00:56:56.260166Z"
+     "iopub.execute_input": "2024-12-31T02:16:19.623806Z",
+     "iopub.status.busy": "2024-12-31T02:16:19.623607Z",
+     "iopub.status.idle": "2024-12-31T02:16:19.887860Z",
+     "shell.execute_reply": "2024-12-31T02:16:19.887386Z"
     }
    },
    "outputs": [
@@ -501,10 +501,10 @@
    "execution_count": 7,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T00:56:56.263028Z",
-     "iopub.status.busy": "2024-12-14T00:56:56.262828Z",
-     "iopub.status.idle": "2024-12-14T00:56:56.270238Z",
-     "shell.execute_reply": "2024-12-14T00:56:56.269664Z"
+     "iopub.execute_input": "2024-12-31T02:16:19.889946Z",
+     "iopub.status.busy": "2024-12-31T02:16:19.889760Z",
+     "iopub.status.idle": "2024-12-31T02:16:19.896929Z",
+     "shell.execute_reply": "2024-12-31T02:16:19.896472Z"
     }
    },
    "outputs": [
@@ -569,10 +569,10 @@
    "execution_count": 8,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T00:56:56.272331Z",
-     "iopub.status.busy": "2024-12-14T00:56:56.272032Z",
-     "iopub.status.idle": "2024-12-14T00:56:56.278433Z",
-     "shell.execute_reply": "2024-12-14T00:56:56.277895Z"
+     "iopub.execute_input": "2024-12-31T02:16:19.898988Z",
+     "iopub.status.busy": "2024-12-31T02:16:19.898588Z",
+     "iopub.status.idle": "2024-12-31T02:16:19.904741Z",
+     "shell.execute_reply": "2024-12-31T02:16:19.904283Z"
     }
    },
    "outputs": [
@@ -618,10 +618,10 @@
    "execution_count": 9,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T00:56:56.280436Z",
-     "iopub.status.busy": "2024-12-14T00:56:56.280126Z",
-     "iopub.status.idle": "2024-12-14T00:56:56.563584Z",
-     "shell.execute_reply": "2024-12-14T00:56:56.562963Z"
+     "iopub.execute_input": "2024-12-31T02:16:19.906624Z",
+     "iopub.status.busy": "2024-12-31T02:16:19.906455Z",
+     "iopub.status.idle": "2024-12-31T02:16:20.185788Z",
+     "shell.execute_reply": "2024-12-31T02:16:20.185241Z"
     }
    },
    "outputs": [],
@@ -644,10 +644,10 @@
    "execution_count": 10,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T00:56:56.566041Z",
-     "iopub.status.busy": "2024-12-14T00:56:56.565857Z",
-     "iopub.status.idle": "2024-12-14T00:56:56.571671Z",
-     "shell.execute_reply": "2024-12-14T00:56:56.571133Z"
+     "iopub.execute_input": "2024-12-31T02:16:20.188423Z",
+     "iopub.status.busy": "2024-12-31T02:16:20.188035Z",
+     "iopub.status.idle": "2024-12-31T02:16:20.194202Z",
+     "shell.execute_reply": "2024-12-31T02:16:20.193723Z"
     }
    },
    "outputs": [
@@ -788,10 +788,10 @@
    "execution_count": 11,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T00:56:56.573659Z",
-     "iopub.status.busy": "2024-12-14T00:56:56.573489Z",
-     "iopub.status.idle": "2024-12-14T00:56:56.577324Z",
-     "shell.execute_reply": "2024-12-14T00:56:56.576752Z"
+     "iopub.execute_input": "2024-12-31T02:16:20.196306Z",
+     "iopub.status.busy": "2024-12-31T02:16:20.195975Z",
+     "iopub.status.idle": "2024-12-31T02:16:20.199953Z",
+     "shell.execute_reply": "2024-12-31T02:16:20.199449Z"
     }
    },
    "outputs": [
@@ -839,10 +839,10 @@
    "execution_count": 12,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T00:56:56.579546Z",
-     "iopub.status.busy": "2024-12-14T00:56:56.579191Z",
-     "iopub.status.idle": "2024-12-14T00:56:56.585331Z",
-     "shell.execute_reply": "2024-12-14T00:56:56.584789Z"
+     "iopub.execute_input": "2024-12-31T02:16:20.202060Z",
+     "iopub.status.busy": "2024-12-31T02:16:20.201729Z",
+     "iopub.status.idle": "2024-12-31T02:16:20.207847Z",
+     "shell.execute_reply": "2024-12-31T02:16:20.207402Z"
     }
    },
    "outputs": [
@@ -895,10 +895,10 @@
    "execution_count": 13,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T00:56:56.587226Z",
-     "iopub.status.busy": "2024-12-14T00:56:56.587069Z",
-     "iopub.status.idle": "2024-12-14T00:56:56.590333Z",
-     "shell.execute_reply": "2024-12-14T00:56:56.589864Z"
+     "iopub.execute_input": "2024-12-31T02:16:20.209900Z",
+     "iopub.status.busy": "2024-12-31T02:16:20.209620Z",
+     "iopub.status.idle": "2024-12-31T02:16:20.212830Z",
+     "shell.execute_reply": "2024-12-31T02:16:20.212288Z"
     }
    },
    "outputs": [],
@@ -940,10 +940,10 @@
    "execution_count": 14,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T00:56:56.592375Z",
-     "iopub.status.busy": "2024-12-14T00:56:56.591968Z",
-     "iopub.status.idle": "2024-12-14T00:56:57.365301Z",
-     "shell.execute_reply": "2024-12-14T00:56:57.364657Z"
+     "iopub.execute_input": "2024-12-31T02:16:20.214983Z",
+     "iopub.status.busy": "2024-12-31T02:16:20.214656Z",
+     "iopub.status.idle": "2024-12-31T02:16:20.977438Z",
+     "shell.execute_reply": "2024-12-31T02:16:20.976839Z"
     }
    },
    "outputs": [
@@ -1929,9 +1929,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"49047634-c443-4608-b07a-015a3c6cf28b\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"49047634-c443-4608-b07a-015a3c6cf28b\")) {                    Plotly.newPlot(                        \"49047634-c443-4608-b07a-015a3c6cf28b\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003ex=%{x}\\u003cbr\\u003ey=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"0_pre\",\"0_mid\",\"1_pre\",\"1_mid\",\"2_pre\",\"2_mid\",\"3_pre\",\"3_mid\",\"4_pre\",\"4_mid\",\"5_pre\",\"5_mid\",\"6_pre\",\"6_mid\",\"7_pre\",\"7_mid\",\"8_pre\",\"8_mid\",\"9_pre\",\"9_mid\",\"10_pre\",\"10_mid\",\"11_pre\",\"11_mid\",\"final_post\"],\"legendgroup\":\"\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[0.0,0.5,1.0,1.5,2.0,2.5,3.0,3.5,4.0,4.5,5.0,5.5,6.0,6.5,7.0,7.5,8.0,8.5,9.0,9.5,10.0,10.5,11.0,11.5,12.0],\"xaxis\":\"x\",\"y\":[1.2936594430357218e-05,-0.0066434419713914394,-0.007525080814957619,-0.009075661189854145,-0.008736724965274334,-0.008685516193509102,-0.006480449344962835,-0.007939981296658516,-0.009662019088864326,-0.01509616058319807,-0.014190619811415672,-0.019929546862840652,-0.009124485775828362,-0.02729814499616623,-0.02985497936606407,0.24972598254680634,0.25055843591690063,0.4500514566898346,0.4599689245223999,5.025448322296143,5.142899036407471,4.73056697845459,4.887067794799805,3.4453935623168945,3.5518839359283447],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"x\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"y\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Logit Difference From Accumulate Residual Stream\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"515a6d53-e13c-4897-a00d-73d0be640d98\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"515a6d53-e13c-4897-a00d-73d0be640d98\")) {                    Plotly.newPlot(                        \"515a6d53-e13c-4897-a00d-73d0be640d98\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003ex=%{x}\\u003cbr\\u003ey=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"0_pre\",\"0_mid\",\"1_pre\",\"1_mid\",\"2_pre\",\"2_mid\",\"3_pre\",\"3_mid\",\"4_pre\",\"4_mid\",\"5_pre\",\"5_mid\",\"6_pre\",\"6_mid\",\"7_pre\",\"7_mid\",\"8_pre\",\"8_mid\",\"9_pre\",\"9_mid\",\"10_pre\",\"10_mid\",\"11_pre\",\"11_mid\",\"final_post\"],\"legendgroup\":\"\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[0.0,0.5,1.0,1.5,2.0,2.5,3.0,3.5,4.0,4.5,5.0,5.5,6.0,6.5,7.0,7.5,8.0,8.5,9.0,9.5,10.0,10.5,11.0,11.5,12.0],\"xaxis\":\"x\",\"y\":[1.2936594430357218e-05,-0.0066434419713914394,-0.007525080814957619,-0.009075661189854145,-0.008736724965274334,-0.008685516193509102,-0.006480449344962835,-0.007939981296658516,-0.009662019088864326,-0.01509616058319807,-0.014190619811415672,-0.019929546862840652,-0.009124485775828362,-0.02729814499616623,-0.02985497936606407,0.24972598254680634,0.25055843591690063,0.4500514566898346,0.4599689245223999,5.025448322296143,5.142899036407471,4.73056697845459,4.887067794799805,3.4453935623168945,3.5518839359283447],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"x\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"y\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Logit Difference From Accumulate Residual Stream\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('49047634-c443-4608-b07a-015a3c6cf28b');\n",
+       "var gd = document.getElementById('515a6d53-e13c-4897-a00d-73d0be640d98');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -1996,10 +1996,10 @@
    "execution_count": 15,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T00:56:57.406173Z",
-     "iopub.status.busy": "2024-12-14T00:56:57.405744Z",
-     "iopub.status.idle": "2024-12-14T00:56:57.439909Z",
-     "shell.execute_reply": "2024-12-14T00:56:57.439283Z"
+     "iopub.execute_input": "2024-12-31T02:16:21.016817Z",
+     "iopub.status.busy": "2024-12-31T02:16:21.016420Z",
+     "iopub.status.idle": "2024-12-31T02:16:21.050138Z",
+     "shell.execute_reply": "2024-12-31T02:16:21.049519Z"
     }
    },
    "outputs": [
@@ -2959,9 +2959,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"619fc2fb-6643-474d-9104-a4fc84996d47\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"619fc2fb-6643-474d-9104-a4fc84996d47\")) {                    Plotly.newPlot(                        \"619fc2fb-6643-474d-9104-a4fc84996d47\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003ex=%{x}\\u003cbr\\u003ey=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"embed\",\"pos_embed\",\"0_attn_out\",\"0_mlp_out\",\"1_attn_out\",\"1_mlp_out\",\"2_attn_out\",\"2_mlp_out\",\"3_attn_out\",\"3_mlp_out\",\"4_attn_out\",\"4_mlp_out\",\"5_attn_out\",\"5_mlp_out\",\"6_attn_out\",\"6_mlp_out\",\"7_attn_out\",\"7_mlp_out\",\"8_attn_out\",\"8_mlp_out\",\"9_attn_out\",\"9_mlp_out\",\"10_attn_out\",\"10_mlp_out\",\"11_attn_out\",\"11_mlp_out\"],\"legendgroup\":\"\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25],\"xaxis\":\"x\",\"y\":[-0.00028366505284793675,0.00029660051222890615,-0.006656361743807793,-0.0008817464113235474,-0.0015505198389291763,0.0003389097983017564,5.121063441038132e-05,0.002205097349360585,-0.001459577470086515,-0.0017220661975443363,-0.0054340846836566925,0.0009054935071617365,-0.005738964769989252,0.010805057361721992,-0.018173594027757645,-0.002556750550866127,0.27958089113235474,0.0008324328809976578,0.19949296116828918,0.009917395189404488,4.565478801727295,0.11744903028011322,-0.41232579946517944,0.1564953327178955,-1.4416704177856445,0.10648898035287857],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"x\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"y\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Logit Difference From Each Layer\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"5929b143-3e55-4982-9cfd-dd66e0b08709\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"5929b143-3e55-4982-9cfd-dd66e0b08709\")) {                    Plotly.newPlot(                        \"5929b143-3e55-4982-9cfd-dd66e0b08709\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003ex=%{x}\\u003cbr\\u003ey=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"embed\",\"pos_embed\",\"0_attn_out\",\"0_mlp_out\",\"1_attn_out\",\"1_mlp_out\",\"2_attn_out\",\"2_mlp_out\",\"3_attn_out\",\"3_mlp_out\",\"4_attn_out\",\"4_mlp_out\",\"5_attn_out\",\"5_mlp_out\",\"6_attn_out\",\"6_mlp_out\",\"7_attn_out\",\"7_mlp_out\",\"8_attn_out\",\"8_mlp_out\",\"9_attn_out\",\"9_mlp_out\",\"10_attn_out\",\"10_mlp_out\",\"11_attn_out\",\"11_mlp_out\"],\"legendgroup\":\"\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25],\"xaxis\":\"x\",\"y\":[-0.00028366505284793675,0.00029660051222890615,-0.006656361743807793,-0.0008817464113235474,-0.0015505198389291763,0.0003389097983017564,5.121063441038132e-05,0.002205097349360585,-0.001459577470086515,-0.0017220661975443363,-0.0054340846836566925,0.0009054935071617365,-0.005738964769989252,0.010805057361721992,-0.018173594027757645,-0.002556750550866127,0.27958089113235474,0.0008324328809976578,0.19949296116828918,0.009917395189404488,4.565478801727295,0.11744903028011322,-0.41232579946517944,0.1564953327178955,-1.4416704177856445,0.10648898035287857],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"x\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"y\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Logit Difference From Each Layer\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('619fc2fb-6643-474d-9104-a4fc84996d47');\n",
+       "var gd = document.getElementById('5929b143-3e55-4982-9cfd-dd66e0b08709');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -3025,10 +3025,10 @@
    "execution_count": 16,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T00:56:57.442269Z",
-     "iopub.status.busy": "2024-12-14T00:56:57.441922Z",
-     "iopub.status.idle": "2024-12-14T00:56:57.860668Z",
-     "shell.execute_reply": "2024-12-14T00:56:57.860098Z"
+     "iopub.execute_input": "2024-12-31T02:16:21.052369Z",
+     "iopub.status.busy": "2024-12-31T02:16:21.052021Z",
+     "iopub.status.idle": "2024-12-31T02:16:21.450791Z",
+     "shell.execute_reply": "2024-12-31T02:16:21.450201Z"
     }
    },
    "outputs": [
@@ -4121,9 +4121,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"d13d12cd-e58c-4918-adf2-4cba0fcd4cfb\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"d13d12cd-e58c-4918-adf2-4cba0fcd4cfb\")) {                    Plotly.newPlot(                        \"d13d12cd-e58c-4918-adf2-4cba0fcd4cfb\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[-0.002056329045444727,-0.0005102002760395408,0.0004685759777203202,0.00012511858949437737,-0.0006028746138326824,-0.00024295759794767946,-0.0023188991472125053,-0.0027583539485931396,0.000564592657610774,0.0009697366622276604,-0.0002504411095287651,4.7396752052009106e-06],[-0.0010071106953546405,0.0003947066143155098,-0.001548723317682743,0.0014034844934940338,-0.0012652697041630745,-0.0011358254123479128,-0.0028159399516880512,-0.0029645331669598818,0.0029190238565206528,0.0025743518490344286,0.00036237656604498625,0.0017548904288560152],[0.0005569161148741841,-0.0011266364017501473,-0.0017354193842038512,-0.0014514740323647857,-0.0002874041674658656,0.0017210585065186024,0.0026659027207642794,0.0031146786641329527,0.0005667298682965338,-0.003666497301310301,-0.0018847138853743672,7.028633262962103e-06],[-0.0007264401647262275,0.0001136981591116637,0.0014301439514383674,0.0007490518037229776,0.002018478699028492,0.0007436765008606017,-0.0004617785452865064,-0.0039057875983417034,0.0011407355777919292,-4.027353134006262e-05,-0.0013293151278048754,-0.0017636835109442472],[-0.0028281747363507748,0.0003364539588801563,-0.0014249025844037533,-0.003777292789891362,0.001599886454641819,0.00029889593133702874,-0.0008046383736655116,0.0020388178527355194,-0.001559373107738793,-0.0006437154370360076,0.001116809668019414,-0.0003500327584333718],[0.0011338141048327088,0.0011259093880653381,-0.0025163700338453054,-0.0014790259301662445,0.00038791849510744214,-6.407558976206928e-05,-0.0005096771637909114,-0.0008841846138238907,0.0006399258272722363,-0.0010097046615555882,-0.006759032607078552,0.0033667325042188168],[-0.015147397294640541,-0.0021350914612412453,0.0025931934360414743,-0.00042673421557992697,-0.005559004843235016,0.002665933221578598,0.0064108846709132195,-0.0038270088844001293,-0.0003842375008389354,-0.0016430213581770658,-0.001334429020062089,-9.183748625218868e-05],[-9.488424984738231e-05,-0.005788922775536776,-0.0006383699947036803,0.134933739900589,-0.0017687629442662,-0.018917974084615707,0.0038733729161322117,-0.0021450738422572613,-0.010327240452170372,0.1832587718963623,-0.0007747872150503099,-0.0010452070273458958],[-0.0038339472375810146,-0.0008046274306252599,-0.01267334446310997,0.008045826107263565,0.003604009747505188,-0.009398158639669418,-0.08272106945514679,0.0035550370812416077,-0.01840413361787796,0.001758674974553287,0.28961312770843506,0.022854033857584],[0.08595199882984161,-0.000693259877152741,0.06816964596509933,0.01311142835766077,-0.02109798789024353,0.05112443491816521,1.384489893913269,0.045837316662073135,-0.038303449749946594,2.9854443073272705,0.001966138370335102,-0.008030208759009838],[0.5608724355697632,0.17083144187927246,-0.033618729561567307,0.05821548402309418,-0.0024530175141990185,0.0018771894974634051,0.2882729470729828,-1.8986413478851318,-0.0015286938287317753,-0.035130057483911514,0.48021769523620605,-0.0009116912842728198],[0.016075868159532547,-0.03986111283302307,-0.3879111707210541,0.01112320739775896,-0.0054778107441961765,-0.002512941136956215,-0.08056112378835678,0.007518645375967026,0.04301108419895172,-0.040082402527332306,-0.9702335000038147,0.011862380430102348]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Each Head\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"8ef8b5c5-d0eb-4f44-b5f8-1221b65bceb0\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"8ef8b5c5-d0eb-4f44-b5f8-1221b65bceb0\")) {                    Plotly.newPlot(                        \"8ef8b5c5-d0eb-4f44-b5f8-1221b65bceb0\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[-0.002056329045444727,-0.0005102002760395408,0.0004685759777203202,0.00012511858949437737,-0.0006028746138326824,-0.00024295759794767946,-0.0023188991472125053,-0.0027583539485931396,0.000564592657610774,0.0009697366622276604,-0.0002504411095287651,4.7396752052009106e-06],[-0.0010071106953546405,0.0003947066143155098,-0.001548723317682743,0.0014034844934940338,-0.0012652697041630745,-0.0011358254123479128,-0.0028159399516880512,-0.0029645331669598818,0.0029190238565206528,0.0025743518490344286,0.00036237656604498625,0.0017548904288560152],[0.0005569161148741841,-0.0011266364017501473,-0.0017354193842038512,-0.0014514740323647857,-0.0002874041674658656,0.0017210585065186024,0.0026659027207642794,0.0031146786641329527,0.0005667298682965338,-0.003666497301310301,-0.0018847138853743672,7.028633262962103e-06],[-0.0007264401647262275,0.0001136981591116637,0.0014301439514383674,0.0007490518037229776,0.002018478699028492,0.0007436765008606017,-0.0004617785452865064,-0.0039057875983417034,0.0011407355777919292,-4.027353134006262e-05,-0.0013293151278048754,-0.0017636835109442472],[-0.0028281747363507748,0.0003364539588801563,-0.0014249025844037533,-0.003777292789891362,0.001599886454641819,0.00029889593133702874,-0.0008046383736655116,0.0020388178527355194,-0.001559373107738793,-0.0006437154370360076,0.001116809668019414,-0.0003500327584333718],[0.0011338141048327088,0.0011259093880653381,-0.0025163700338453054,-0.0014790259301662445,0.00038791849510744214,-6.407558976206928e-05,-0.0005096771637909114,-0.0008841846138238907,0.0006399258272722363,-0.0010097046615555882,-0.006759032607078552,0.0033667325042188168],[-0.015147397294640541,-0.0021350914612412453,0.0025931934360414743,-0.00042673421557992697,-0.005559004843235016,0.002665933221578598,0.0064108846709132195,-0.0038270088844001293,-0.0003842375008389354,-0.0016430213581770658,-0.001334429020062089,-9.183748625218868e-05],[-9.488424984738231e-05,-0.005788922775536776,-0.0006383699947036803,0.134933739900589,-0.0017687629442662,-0.018917974084615707,0.0038733729161322117,-0.0021450738422572613,-0.010327240452170372,0.1832587718963623,-0.0007747872150503099,-0.0010452070273458958],[-0.0038339472375810146,-0.0008046274306252599,-0.01267334446310997,0.008045826107263565,0.003604009747505188,-0.009398158639669418,-0.08272106945514679,0.0035550370812416077,-0.01840413361787796,0.001758674974553287,0.28961312770843506,0.022854033857584],[0.08595199882984161,-0.000693259877152741,0.06816964596509933,0.01311142835766077,-0.02109798789024353,0.05112443491816521,1.384489893913269,0.045837316662073135,-0.038303449749946594,2.9854443073272705,0.001966138370335102,-0.008030208759009838],[0.5608724355697632,0.17083144187927246,-0.033618729561567307,0.05821548402309418,-0.0024530175141990185,0.0018771894974634051,0.2882729470729828,-1.8986413478851318,-0.0015286938287317753,-0.035130057483911514,0.48021769523620605,-0.0009116912842728198],[0.016075868159532547,-0.03986111283302307,-0.3879111707210541,0.01112320739775896,-0.0054778107441961765,-0.002512941136956215,-0.08056112378835678,0.007518645375967026,0.04301108419895172,-0.040082402527332306,-0.9702335000038147,0.011862380430102348]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Each Head\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('d13d12cd-e58c-4918-adf2-4cba0fcd4cfb');\n",
+       "var gd = document.getElementById('8ef8b5c5-d0eb-4f44-b5f8-1221b65bceb0');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -4195,10 +4195,10 @@
    "execution_count": 17,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T00:56:57.863192Z",
-     "iopub.status.busy": "2024-12-14T00:56:57.862860Z",
-     "iopub.status.idle": "2024-12-14T00:56:57.868039Z",
-     "shell.execute_reply": "2024-12-14T00:56:57.867562Z"
+     "iopub.execute_input": "2024-12-31T02:16:21.453082Z",
+     "iopub.status.busy": "2024-12-31T02:16:21.452751Z",
+     "iopub.status.idle": "2024-12-31T02:16:21.458007Z",
+     "shell.execute_reply": "2024-12-31T02:16:21.457461Z"
     }
    },
    "outputs": [],
@@ -4263,29 +4263,29 @@
    "execution_count": 18,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T00:56:57.870050Z",
-     "iopub.status.busy": "2024-12-14T00:56:57.869714Z",
-     "iopub.status.idle": "2024-12-14T00:56:57.879293Z",
-     "shell.execute_reply": "2024-12-14T00:56:57.878713Z"
+     "iopub.execute_input": "2024-12-31T02:16:21.460124Z",
+     "iopub.status.busy": "2024-12-31T02:16:21.459821Z",
+     "iopub.status.idle": "2024-12-31T02:16:21.469233Z",
+     "shell.execute_reply": "2024-12-31T02:16:21.468692Z"
     }
    },
    "outputs": [
     {
      "data": {
       "text/html": [
-       "<div style='max-width: 700px;'><h2>Top 3 Positive Logit Attribution Heads</h2><br/><div id=\"circuits-vis-238275f7-4d84\" style=\"margin: 15px 0;\"/>\n",
+       "<div style='max-width: 700px;'><h2>Top 3 Positive Logit Attribution Heads</h2><br/><div id=\"circuits-vis-3a6547db-fa7c\" style=\"margin: 15px 0;\"/>\n",
        "    <script crossorigin type=\"module\">\n",
        "    import { render, AttentionHeads } from \"https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js\";\n",
        "    render(\n",
-       "      \"circuits-vis-238275f7-4d84\",\n",
+       "      \"circuits-vis-3a6547db-fa7c\",\n",
        "      AttentionHeads,\n",
        "      {\"attention\": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9979304075241089, 0.002069620881229639, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9971864819526672, 0.0010516609763726592, 0.0017618348356336355, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9594857096672058, 0.001310725579969585, 0.03694308176636696, 0.002260456094518304, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9891613125801086, 0.001052629784680903, 0.0048541901633143425, 0.001104357186704874, 0.003827564185485244, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9963598847389221, 0.0007981774397194386, 0.0007764195324853063, 0.00019262291607446969, 0.0002416159404674545, 0.0016313291853293777, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.981286346912384, 0.001863775309175253, 0.006344134453684092, 0.0002960922720376402, 0.004604941233992577, 0.0013966941041871905, 0.00420788861811161, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9950011372566223, 0.0013230532640591264, 0.0007920759962871671, 0.00024644460063427687, 0.0003405151073820889, 0.00016826140927150846, 0.0002837753272615373, 0.0018447580514475703, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9920291900634766, 0.0013539871433749795, 0.0007168339216150343, 9.214139572577551e-05, 0.00013419234892353415, 0.00019706363673321903, 0.00035278062568977475, 0.0002426079590804875, 0.004881155211478472, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5483196377754211, 0.0075838202610611916, 0.28265973925590515, 0.0029685653280466795, 0.14815661311149597, 0.0008172960951924324, 0.0006479909643530846, 0.0017078499076887965, 0.004129356704652309, 0.003009046893566847, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8642559051513672, 0.0036493062507361174, 0.004925842396914959, 0.06957440823316574, 0.038683053106069565, 0.009609837085008621, 0.0008548588375560939, 0.0005643004551529884, 0.003707138355821371, 0.0013952680164948106, 0.002780066104605794, 0.0, 0.0, 0.0, 0.0], [0.30252957344055176, 0.006584456656128168, 0.1401754468679428, 0.030065739527344704, 0.4806697964668274, 0.0005923578282818198, 0.0005309724947437644, 0.0024134721606969833, 0.008505754172801971, 0.0013766245683655143, 0.023764921352267265, 0.0027908117044717073, 0.0, 0.0, 0.0], [0.7773900032043457, 0.003722917754203081, 0.022378109395503998, 0.015139483846724033, 0.017408354207873344, 0.002416662173345685, 0.000709561922121793, 0.0007399603491649032, 0.13850137591362, 0.0023606533650308847, 0.005776867736130953, 0.0028014343697577715, 0.010654616169631481, 0.0, 0.0], [0.9735962152481079, 0.0012812522472813725, 0.002617918187752366, 9.891873924061656e-05, 0.0005098494002595544, 0.00012000725837424397, 0.00045229491661302745, 0.00010779645526781678, 0.002954537281766534, 0.001644388772547245, 0.0012141242623329163, 0.00018637391622178257, 0.0008510759216733277, 0.01436527632176876, 0.0], [0.0990300253033638, 0.000971083587501198, 0.06413238495588303, 0.005246965680271387, 0.8110400438308716, 8.250321843661368e-05, 6.596777529921383e-05, 0.0011991349747404456, 0.0008852760074660182, 0.00024891181965358555, 0.010336406528949738, 7.952339365147054e-05, 0.0049433172680437565, 0.00028918671887367964, 0.0014492359478026628]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9985041618347168, 0.0014959010295569897, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9966639876365662, 0.00046957843005657196, 0.002866449998691678, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9208283424377441, 0.0007845019572414458, 0.07676984369754791, 0.001617342815734446, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9835838675498962, 0.0006018307176418602, 0.0030122774187475443, 0.006188061088323593, 0.0066139851696789265, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9950478076934814, 0.0007798039587214589, 0.0007059435010887682, 0.0003824840241577476, 0.0007798775332048535, 0.0023041421081870794, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9838821887969971, 0.0015907409833744168, 0.0063361418433487415, 0.0004584550915751606, 0.004829846788197756, 0.0009966548532247543, 0.0019060741178691387, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.995481014251709, 0.0010436347220093012, 0.0005037460359744728, 0.0005756175960414112, 0.00046194688184186816, 0.0004267194017302245, 0.0006751787732355297, 0.0008320942288264632, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9798213243484497, 0.008518284186720848, 0.0010190301109105349, 0.00029932931647635996, 0.0003097986045759171, 0.0003330526524223387, 0.001541984616778791, 0.0008126517641358078, 0.007344536483287811, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6489241123199463, 0.008722440339624882, 0.10265976935625076, 0.0038905434776097536, 0.22694909572601318, 0.0011350901331752539, 0.00045881845289841294, 0.0004216691595502198, 0.0053738984279334545, 0.0014645822811871767, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9156036972999573, 0.0013435552828013897, 0.0011673923581838608, 0.04532041400671005, 0.020157871767878532, 0.007576430682092905, 0.0011305580846965313, 0.00024880640557967126, 0.004846310243010521, 0.0007249554619193077, 0.0018800381803885102, 0.0, 0.0, 0.0, 0.0], [0.4962114989757538, 0.0019782893359661102, 0.05844113230705261, 0.00783691368997097, 0.40375715494155884, 0.0005728096584789455, 0.0002963987353723496, 0.0009752871701493859, 0.0046507869847118855, 0.00032523463596589863, 0.02318551018834114, 0.0017689995002001524, 0.0, 0.0, 0.0], [0.8397698402404785, 0.0018636900931596756, 0.005577529780566692, 0.020380759611725807, 0.0292733795940876, 0.004448764491826296, 0.0011503315763548017, 0.00024191653938032687, 0.08715140074491501, 0.0013588638976216316, 0.0026416215114295483, 0.0019266613526269794, 0.004215260501950979, 0.0, 0.0], [0.9744170308113098, 0.0024021149147301912, 0.00455052312463522, 0.00020547708845697343, 0.002662503393366933, 0.00021827162709087133, 0.0008242498734034598, 0.0004054978198837489, 0.002643037121742964, 0.0015235628234222531, 0.0031088448595255613, 7.676492532482371e-05, 0.0018502527382224798, 0.0051117767579853535, 0.0], [0.18470405042171478, 0.00038850627606734633, 0.06518032401800156, 0.0014534658985212445, 0.7318763732910156, 0.00010888298129430041, 2.748575025179889e-05, 0.00011231198004679754, 0.0013672056375071406, 4.0162569348467514e-05, 0.012543701566755772, 5.95860110479407e-05, 0.0007756491540931165, 0.0010575354099273682, 0.00030486442847177386]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9776753783226013, 0.022324683144688606, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9430753588676453, 0.018823042511940002, 0.038101643323898315, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8963208794593811, 0.006171742454171181, 0.07676312327384949, 0.02074429951608181, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8522229790687561, 0.017844438552856445, 0.05478672310709953, 0.012976272031664848, 0.062169551849365234, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9669023752212524, 0.004049547016620636, 0.002062328392639756, 0.007766399998217821, 0.007081815041601658, 0.012137639336287975, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7676379680633545, 0.018502332270145416, 0.045013245195150375, 0.012471744790673256, 0.10591386258602142, 0.018323613330721855, 0.03213733434677124, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.891828179359436, 0.010069617070257664, 0.013090009801089764, 0.010685192421078682, 0.031169477850198746, 0.006502270698547363, 0.02103857509791851, 0.015616719610989094, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9457208514213562, 0.0045005157589912415, 0.013905955478549004, 0.0013248942559584975, 0.01557664293795824, 0.001115700462833047, 0.0007837467710487545, 0.0007244577864184976, 0.016347244381904602, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5131928324699402, 0.006032112520188093, 0.11406763643026352, 0.009519217535853386, 0.30613386631011963, 0.0034422571770846844, 0.002601674757897854, 0.002665397012606263, 0.03412351757287979, 0.008221502415835857, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8395131230354309, 0.008326224982738495, 0.012920893728733063, 0.01282776053994894, 0.029934609308838844, 0.013884111307561398, 0.0034839101135730743, 0.0018691613804548979, 0.039465710520744324, 0.011843967251479626, 0.025930611416697502, 0.0, 0.0, 0.0, 0.0], [0.3585011661052704, 0.004236206877976656, 0.10680253803730011, 0.015598620288074017, 0.41607996821403503, 0.002491925610229373, 0.003585703205317259, 0.0038999069947749376, 0.015854908153414726, 0.005938141606748104, 0.062027692794799805, 0.004983273800462484, 0.0, 0.0, 0.0], [0.612542450428009, 0.004679449368268251, 0.029155218973755836, 0.028240399435162544, 0.08258962631225586, 0.007909866981208324, 0.011129351332783699, 0.0034775957465171814, 0.17027747631072998, 0.010538897477090359, 0.015773506835103035, 0.013295088894665241, 0.01039104349911213, 0.0, 0.0], [0.875810444355011, 0.007089323364198208, 0.01208286453038454, 0.0025996132753789425, 0.021269435063004494, 0.00165357009973377, 0.0019036760786548257, 0.0016310046194121242, 0.02823040634393692, 0.008473552763462067, 0.011080354452133179, 0.002760963048785925, 0.0025940914638340473, 0.02282082475721836, 0.0], [0.5415229201316833, 0.003397508757188916, 0.03888256847858429, 0.007577123120427132, 0.3613935708999634, 0.001381068374030292, 0.0010475177550688386, 0.0012340175453573465, 0.006947703659534454, 0.003757019527256489, 0.0176718570291996, 0.0011743978830054402, 0.0036232525017112494, 0.00219246419146657, 0.008196980692446232]]], \"attentionHeadNames\": [\"L9H9\", \"L9H6\", \"L10H0\"], \"tokens\": [\"<|endoftext|>\", \"When\", \" John\", \" and\", \" Mary\", \" went\", \" to\", \" the\", \" shops\", \",\", \" John\", \" gave\", \" the\", \" bag\", \" to\"]}\n",
        "    )\n",
-       "    </script></div><div style='max-width: 700px;'><h2>Top 3 Negative Logit Attribution Heads</h2><br/><div id=\"circuits-vis-a59ac87b-fe56\" style=\"margin: 15px 0;\"/>\n",
+       "    </script></div><div style='max-width: 700px;'><h2>Top 3 Negative Logit Attribution Heads</h2><br/><div id=\"circuits-vis-f26fcf72-5687\" style=\"margin: 15px 0;\"/>\n",
        "    <script crossorigin type=\"module\">\n",
        "    import { render, AttentionHeads } from \"https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js\";\n",
        "    render(\n",
-       "      \"circuits-vis-a59ac87b-fe56\",\n",
+       "      \"circuits-vis-f26fcf72-5687\",\n",
        "      AttentionHeads,\n",
        "      {\"attention\": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9628916382789612, 0.03710832819342613, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9778335094451904, 0.0034864130429923534, 0.018680110573768616, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8737697601318359, 0.005964328069239855, 0.08126737922430038, 0.03899851813912392, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8745138645172119, 0.017276111990213394, 0.018874678760766983, 0.054268334060907364, 0.035066962242126465, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9704693555831909, 0.0035177830141037703, 0.0003239834331907332, 0.004373463336378336, 0.0003953034174628556, 0.02092011272907257, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7694658041000366, 0.006685878615826368, 0.011966485530138016, 0.040966276079416275, 0.03344443440437317, 0.04401002824306488, 0.0934610366821289, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8177722692489624, 0.009112595580518246, 0.0042336746118962765, 0.03900325670838356, 0.012653318233788013, 0.007337852846831083, 0.07410021126270294, 0.0357869453728199, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8309933543205261, 0.018796611577272415, 0.00330327614210546, 0.01983213610947132, 0.004009497817605734, 0.012078307569026947, 0.03246007487177849, 0.010322270914912224, 0.06820447742938995, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.31403300166130066, 0.01522789802402258, 0.19789263606071472, 0.059379592537879944, 0.3236042559146881, 0.009037697687745094, 0.019917670637369156, 0.012475269846618176, 0.013939663767814636, 0.03449229896068573, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7319568991661072, 0.03087025322020054, 0.005064091179519892, 0.08703826367855072, 0.004982746671885252, 0.044725559651851654, 0.02350614219903946, 0.0063977413810789585, 0.0034696452785283327, 0.048653144389390945, 0.013335512951016426, 0.0, 0.0, 0.0, 0.0], [0.13710583746433258, 0.012017901986837387, 0.14592327177524567, 0.015683427453041077, 0.44125232100486755, 0.004342416767030954, 0.011117888614535332, 0.007466601673513651, 0.0012341630645096302, 0.01946023479104042, 0.16842849552631378, 0.035967420786619186, 0.0, 0.0, 0.0], [0.4052484929561615, 0.009720955975353718, 0.03158125653862953, 0.09247038513422012, 0.048808369785547256, 0.008174203336238861, 0.031708307564258575, 0.02787904627621174, 0.14128026366233826, 0.0302340779453516, 0.03392859548330307, 0.03171432018280029, 0.10725171118974686, 0.0, 0.0], [0.808469295501709, 0.0056702690199017525, 0.006755992770195007, 0.008696584962308407, 0.010473440401256084, 0.003464736510068178, 0.006760997697710991, 0.0025889205280691385, 0.015097705647349358, 0.01603602059185505, 0.011288060806691647, 0.017277495935559273, 0.00779919745400548, 0.07962135970592499, 0.0], [0.02939111366868019, 0.0016060526249930263, 0.06005561351776123, 0.008175727911293507, 0.8099164366722107, 0.00039796155760996044, 0.0028282543644309044, 0.0033632616978138685, 0.0001569116284372285, 0.0036459483671933413, 0.05494074895977974, 0.001666039228439331, 0.015368940308690071, 5.016689101466909e-05, 0.008436810225248337]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9124024510383606, 0.08759760111570358, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.97407466173172, 0.01434413529932499, 0.011581187136471272, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8240247964859009, 0.03700513392686844, 0.06643965095281601, 0.07253041118383408, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5370795726776123, 0.15483324229717255, 0.0903128981590271, 0.12131187319755554, 0.0964624434709549, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9388002157211304, 0.028733234852552414, 0.0003431019140407443, 0.010481251403689384, 0.000839930260553956, 0.02080223336815834, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6318807005882263, 0.059302832931280136, 0.03517911583185196, 0.0317409411072731, 0.1174689382314682, 0.06869616359472275, 0.05573126673698425, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7625216841697693, 0.04435169696807861, 0.010552966967225075, 0.034276288002729416, 0.014203979633748531, 0.012703591957688332, 0.07259687036275864, 0.048792947083711624, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5725991725921631, 0.12681783735752106, 0.03727060556411743, 0.0431414395570755, 0.04129580408334732, 0.046447426080703735, 0.029164383187890053, 0.03751189261674881, 0.06575141847133636, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12316936254501343, 0.026347072795033455, 0.27960509061813354, 0.041989635676145554, 0.44105827808380127, 0.006138973869383335, 0.007793003693223, 0.007175281643867493, 0.011286810971796513, 0.055436521768569946, 0.0, 0.0, 0.0, 0.0, 0.0], [0.44855940341949463, 0.1432485729455948, 0.010774216614663601, 0.07855041325092316, 0.01404787041246891, 0.014864671975374222, 0.008633698336780071, 0.0030146560166031122, 0.012792127206921577, 0.16408587992191315, 0.10142850130796432, 0.0, 0.0, 0.0, 0.0], [0.10228201746940613, 0.0238832775503397, 0.055250268429517746, 0.0520680733025074, 0.5759841799736023, 0.0011948152678087354, 0.0013662497512996197, 0.0025677781086415052, 0.0018699930515140295, 0.043160032480955124, 0.10887196660041809, 0.03150142729282379, 0.0, 0.0, 0.0], [0.39313194155693054, 0.04406267777085304, 0.04009911045432091, 0.07407406717538834, 0.04061643034219742, 0.007346798665821552, 0.009507199749350548, 0.013865168206393719, 0.0640539675951004, 0.04894229769706726, 0.0627753809094429, 0.0999651551246643, 0.10155977308750153, 0.0, 0.0], [0.7573198676109314, 0.015095721930265427, 0.007350177504122257, 0.013156400062143803, 0.005726401228457689, 0.008556416258215904, 0.008282607421278954, 0.010307075455784798, 0.002904881490394473, 0.017329057678580284, 0.015188980847597122, 0.0796574279665947, 0.030244464054703712, 0.02888045459985733, 0.0], [0.08579524606466293, 0.023773644119501114, 0.028759371489286423, 0.050303805619478226, 0.674644947052002, 0.0010763936443254352, 0.0011514866491779685, 0.002721975091844797, 0.0005617713322862983, 0.039091579616069794, 0.04861969128251076, 0.015520088374614716, 0.0040845321491360664, 0.00043291584006510675, 0.023462524637579918]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9691730737686157, 0.03082684986293316, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9864587187767029, 0.0076598357409238815, 0.00588154187425971, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9318376183509827, 0.011915593408048153, 0.02053167298436165, 0.03571505844593048, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8786704540252686, 0.03427596017718315, 0.01315787062048912, 0.02705124393105507, 0.04684450104832649, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9850097298622131, 0.00501946872100234, 0.0013920071069151163, 0.0018760499078780413, 0.004171107430011034, 0.002531560370698571, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8907089829444885, 0.02026253007352352, 0.020904401317238808, 0.009134626016020775, 0.04057806357741356, 0.003215113654732704, 0.015196278691291809, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9318532347679138, 0.012423796579241753, 0.007052858360111713, 0.00626762630417943, 0.009715708903968334, 0.0013882736675441265, 0.012697475962340832, 0.018601125106215477, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7722940444946289, 0.03829518333077431, 0.03943629562854767, 0.00972924567759037, 0.11019386351108551, 0.009773456491529942, 0.00886810664087534, 0.003977742046117783, 0.007432003039866686, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6659272909164429, 0.019893741235136986, 0.05748209357261658, 0.026046503335237503, 0.1897869110107422, 0.001734035206027329, 0.004116130527108908, 0.004545124247670174, 0.007496052421629429, 0.022972112521529198, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8588331341743469, 0.01852157525718212, 0.010080324485898018, 0.0161746833473444, 0.044801417738199234, 0.018235526978969574, 0.0026932875625789165, 0.0009044647449627519, 0.004338268656283617, 0.01020037941634655, 0.015216901898384094, 0.0, 0.0, 0.0, 0.0], [0.7452806234359741, 0.02059219218790531, 0.033448558300733566, 0.04560310021042824, 0.09397682547569275, 0.0019108005799353123, 0.0039161862805485725, 0.004672198090702295, 0.003216084325686097, 0.0180679801851511, 0.0163005068898201, 0.013014941476285458, 0.0, 0.0, 0.0], [0.7450795769691467, 0.016002941876649857, 0.02564326673746109, 0.03000519424676895, 0.06691146641969681, 0.0032243167515844107, 0.006530732847750187, 0.006865624338388443, 0.04355334863066673, 0.012662945315241814, 0.010547308251261711, 0.010315663181245327, 0.022657567635178566, 0.0, 0.0], [0.8363587260246277, 0.013512525707483292, 0.01655869372189045, 0.008880337700247765, 0.05630703642964363, 0.0022557524498552084, 0.0023984003346413374, 0.0012933476828038692, 0.003150224220007658, 0.013540910556912422, 0.02165023609995842, 0.005392791703343391, 0.00432277237996459, 0.014378219842910767, 0.0], [0.7888551354408264, 0.014087573625147343, 0.022160783410072327, 0.04220619052648544, 0.06532405316829681, 0.0011144893942400813, 0.001956802560016513, 0.002514239400625229, 0.001448699738830328, 0.012026125565171242, 0.009607555344700813, 0.006895523052662611, 0.004557081963866949, 0.003263341262936592, 0.023982450366020203]]], \"attentionHeadNames\": [\"L10H7\", \"L11H10\", \"L11H2\"], \"tokens\": [\"<|endoftext|>\", \"When\", \" John\", \" and\", \" Mary\", \" went\", \" to\", \" the\", \" shops\", \",\", \" John\", \" gave\", \" the\", \" bag\", \" to\"]}\n",
        "    )\n",
@@ -4414,10 +4414,10 @@
    "execution_count": 19,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T00:56:57.881314Z",
-     "iopub.status.busy": "2024-12-14T00:56:57.881146Z",
-     "iopub.status.idle": "2024-12-14T00:56:58.167932Z",
-     "shell.execute_reply": "2024-12-14T00:56:58.167251Z"
+     "iopub.execute_input": "2024-12-31T02:16:21.471481Z",
+     "iopub.status.busy": "2024-12-31T02:16:21.471163Z",
+     "iopub.status.idle": "2024-12-31T02:16:21.746065Z",
+     "shell.execute_reply": "2024-12-31T02:16:21.745472Z"
     }
    },
    "outputs": [
@@ -4449,10 +4449,10 @@
    "execution_count": 20,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T00:56:58.170608Z",
-     "iopub.status.busy": "2024-12-14T00:56:58.170212Z",
-     "iopub.status.idle": "2024-12-14T00:56:58.175091Z",
-     "shell.execute_reply": "2024-12-14T00:56:58.174594Z"
+     "iopub.execute_input": "2024-12-31T02:16:21.748465Z",
+     "iopub.status.busy": "2024-12-31T02:16:21.748056Z",
+     "iopub.status.idle": "2024-12-31T02:16:21.752399Z",
+     "shell.execute_reply": "2024-12-31T02:16:21.751867Z"
     }
    },
    "outputs": [
@@ -4493,10 +4493,10 @@
    "execution_count": 21,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T00:56:58.177256Z",
-     "iopub.status.busy": "2024-12-14T00:56:58.176915Z",
-     "iopub.status.idle": "2024-12-14T00:57:45.079131Z",
-     "shell.execute_reply": "2024-12-14T00:57:45.078479Z"
+     "iopub.execute_input": "2024-12-31T02:16:21.754463Z",
+     "iopub.status.busy": "2024-12-31T02:16:21.754163Z",
+     "iopub.status.idle": "2024-12-31T02:17:07.861445Z",
+     "shell.execute_reply": "2024-12-31T02:17:07.860794Z"
     }
    },
    "outputs": [],
@@ -4554,10 +4554,10 @@
    "execution_count": 22,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T00:57:45.081699Z",
-     "iopub.status.busy": "2024-12-14T00:57:45.081519Z",
-     "iopub.status.idle": "2024-12-14T00:57:45.113039Z",
-     "shell.execute_reply": "2024-12-14T00:57:45.112455Z"
+     "iopub.execute_input": "2024-12-31T02:17:07.864035Z",
+     "iopub.status.busy": "2024-12-31T02:17:07.863857Z",
+     "iopub.status.idle": "2024-12-31T02:17:07.894533Z",
+     "shell.execute_reply": "2024-12-31T02:17:07.894001Z"
     }
    },
    "outputs": [
@@ -5696,9 +5696,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"4bc9653c-0aeb-4930-9e54-0fc92a4e0591\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"4bc9653c-0aeb-4930-9e54-0fc92a4e0591\")) {                    Plotly.newPlot(                        \"4bc9653c-0aeb-4930-9e54-0fc92a4e0591\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"x\":[\"\\u003c|endoftext|\\u003e_0\",\"When_1\",\" John_2\",\" and_3\",\" Mary_4\",\" went_5\",\" to_6\",\" the_7\",\" shops_8\",\",_9\",\" John_10\",\" gave_11\",\" the_12\",\" bag_13\",\" to_14\"],\"z\":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.00065016746521,-0.00024725322145968676,9.061812306754291e-06,-0.00036435198853723705,-4.832966806134209e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0010517835617065,-2.6816253011929803e-05,-2.0540108380373567e-05,-0.0004592325130943209,-0.0005939850234426558],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0002665519714355,0.0008679538150317967,0.0005159862921573222,-0.0009933760156854987,-0.0008652352844364941],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9949080944061279,0.005429603159427643,0.0016055518062785268,-0.0006179149495437741,-0.0016324687749147415],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9675664305686951,0.03134222328662872,0.0028418514411896467,-0.0012303927214816213,-0.0009862943552434444],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9675208926200867,0.031000729650259018,0.001782458508387208,-0.0004856795712839812,-0.000646778498776257],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9228320121765137,0.05134553834795952,0.004729225765913725,0.0009345413418486714,0.017047081142663956],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.6565485000610352,0.02385673113167286,0.002357447287067771,-1.7318130630883388e-05,0.3186914026737213],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.027302434667944908,0.03142485395073891,0.0018206859240308404,0.0007993190083652735,0.9383869171142578],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.026841893792152405,0.020981015637516975,0.0012513356050476432,0.0003238087520003319,1.0048280954360962],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.005687932018190622,0.014263695105910301,0.0004871227720286697,-8.984619489638135e-05,0.9914218783378601]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Position: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Position\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Patched Residual Stream\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"aa440a00-8bb4-44ef-8d9b-a548c3b23622\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"aa440a00-8bb4-44ef-8d9b-a548c3b23622\")) {                    Plotly.newPlot(                        \"aa440a00-8bb4-44ef-8d9b-a548c3b23622\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"x\":[\"\\u003c|endoftext|\\u003e_0\",\"When_1\",\" John_2\",\" and_3\",\" Mary_4\",\" went_5\",\" to_6\",\" the_7\",\" shops_8\",\",_9\",\" John_10\",\" gave_11\",\" the_12\",\" bag_13\",\" to_14\"],\"z\":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.00065016746521,-0.00024725322145968676,9.061812306754291e-06,-0.00036435198853723705,-4.832966806134209e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0010517835617065,-2.6816253011929803e-05,-2.0540108380373567e-05,-0.0004592325130943209,-0.0005939850234426558],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0002665519714355,0.0008679538150317967,0.0005159862921573222,-0.0009933760156854987,-0.0008652352844364941],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9949080944061279,0.005429603159427643,0.0016055518062785268,-0.0006179149495437741,-0.0016324687749147415],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9675664305686951,0.03134222328662872,0.0028418514411896467,-0.0012303927214816213,-0.0009862943552434444],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9675208926200867,0.031000729650259018,0.001782458508387208,-0.0004856795712839812,-0.000646778498776257],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9228320121765137,0.05134553834795952,0.004729225765913725,0.0009345413418486714,0.017047081142663956],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.6565485000610352,0.02385673113167286,0.002357447287067771,-1.7318130630883388e-05,0.3186914026737213],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.027302434667944908,0.03142485395073891,0.0018206859240308404,0.0007993190083652735,0.9383869171142578],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.026841893792152405,0.020981015637516975,0.0012513356050476432,0.0003238087520003319,1.0048280954360962],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.005687932018190622,0.014263695105910301,0.0004871227720286697,-8.984619489638135e-05,0.9914218783378601]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Position: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Position\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Patched Residual Stream\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('4bc9653c-0aeb-4930-9e54-0fc92a4e0591');\n",
+       "var gd = document.getElementById('aa440a00-8bb4-44ef-8d9b-a548c3b23622');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -5758,10 +5758,10 @@
    "execution_count": 23,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T00:57:45.115204Z",
-     "iopub.status.busy": "2024-12-14T00:57:45.114883Z",
-     "iopub.status.idle": "2024-12-14T00:59:19.604005Z",
-     "shell.execute_reply": "2024-12-14T00:59:19.603315Z"
+     "iopub.execute_input": "2024-12-31T02:17:07.896717Z",
+     "iopub.status.busy": "2024-12-31T02:17:07.896398Z",
+     "iopub.status.idle": "2024-12-31T02:18:39.900436Z",
+     "shell.execute_reply": "2024-12-31T02:18:39.899781Z"
     }
    },
    "outputs": [],
@@ -5812,10 +5812,10 @@
    "execution_count": 24,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T00:59:19.606789Z",
-     "iopub.status.busy": "2024-12-14T00:59:19.606567Z",
-     "iopub.status.idle": "2024-12-14T00:59:19.637315Z",
-     "shell.execute_reply": "2024-12-14T00:59:19.636727Z"
+     "iopub.execute_input": "2024-12-31T02:18:39.903353Z",
+     "iopub.status.busy": "2024-12-31T02:18:39.902895Z",
+     "iopub.status.idle": "2024-12-31T02:18:39.932945Z",
+     "shell.execute_reply": "2024-12-31T02:18:39.932449Z"
     }
    },
    "outputs": [
@@ -6954,9 +6954,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"191ca322-bcdc-49ad-a055-e5a33b6bafa3\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"191ca322-bcdc-49ad-a055-e5a33b6bafa3\")) {                    Plotly.newPlot(                        \"191ca322-bcdc-49ad-a055-e5a33b6bafa3\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"x\":[\"\\u003c|endoftext|\\u003e_0\",\"When_1\",\" John_2\",\" and_3\",\" Mary_4\",\" went_5\",\" to_6\",\" the_7\",\" shops_8\",\",_9\",\" John_10\",\" gave_11\",\" the_12\",\" bag_13\",\" to_14\"],\"z\":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.03545692563056946,-0.00024725322145968676,9.061812306754291e-06,-0.00036435198853723705,-4.832966806134209e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0029837191104888916,7.950901635922492e-05,2.097641845466569e-05,8.054944191826507e-05,-0.0005966364406049252],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0019127808045595884,0.000666748033836484,0.00039496077806688845,-0.0007048076367937028,-0.0002728612453211099],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.15463140606880188,0.0038022694643586874,0.0005171945667825639,-0.00012015292304567993,-0.0005605234182439744],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.005405941512435675,0.019581804051995277,0.0010072707664221525,-0.00024272232258226722,0.0007941504009068012],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.35209786891937256,0.0010532510932534933,0.00022453156998381019,0.00013320863945409656,8.259674359578639e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.11985944956541061,0.02124394290149212,0.002728075487539172,0.0013410811079666018,0.017973870038986206],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.013310795649886131,0.011509508825838566,0.00037489054375328124,-4.121446545468643e-05,0.2976022958755493],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.001500166254118085,0.017352566123008728,0.0005847889697179198,0.0010113989701494575,0.5697317123413086],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0001285099278902635,0.006301181390881538,0.00014156564429868013,0.00031266608857549727,0.2715233862400055],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0009370585321448743,8.679202437633649e-05,0.00033203151542693377,1.543864300401765e-06,-0.1929759979248047],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.4061770737171173]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Position: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Position\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Patched Attention Layer\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"d488530c-df0a-4dd0-a5d4-bd55b79cf332\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"d488530c-df0a-4dd0-a5d4-bd55b79cf332\")) {                    Plotly.newPlot(                        \"d488530c-df0a-4dd0-a5d4-bd55b79cf332\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"x\":[\"\\u003c|endoftext|\\u003e_0\",\"When_1\",\" John_2\",\" and_3\",\" Mary_4\",\" went_5\",\" to_6\",\" the_7\",\" shops_8\",\",_9\",\" John_10\",\" gave_11\",\" the_12\",\" bag_13\",\" to_14\"],\"z\":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.03545692563056946,-0.00024725322145968676,9.061812306754291e-06,-0.00036435198853723705,-4.832966806134209e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0029837191104888916,7.950901635922492e-05,2.097641845466569e-05,8.054944191826507e-05,-0.0005966364406049252],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0019127808045595884,0.000666748033836484,0.00039496077806688845,-0.0007048076367937028,-0.0002728612453211099],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.15463140606880188,0.0038022694643586874,0.0005171945667825639,-0.00012015292304567993,-0.0005605234182439744],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.005405941512435675,0.019581804051995277,0.0010072707664221525,-0.00024272232258226722,0.0007941504009068012],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.35209786891937256,0.0010532510932534933,0.00022453156998381019,0.00013320863945409656,8.259674359578639e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.11985944956541061,0.02124394290149212,0.002728075487539172,0.0013410811079666018,0.017973870038986206],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.013310795649886131,0.011509508825838566,0.00037489054375328124,-4.121446545468643e-05,0.2976022958755493],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.001500166254118085,0.017352566123008728,0.0005847889697179198,0.0010113989701494575,0.5697317123413086],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0001285099278902635,0.006301181390881538,0.00014156564429868013,0.00031266608857549727,0.2715233862400055],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0009370585321448743,8.679202437633649e-05,0.00033203151542693377,1.543864300401765e-06,-0.1929759979248047],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.4061770737171173]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Position: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Position\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Patched Attention Layer\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('191ca322-bcdc-49ad-a055-e5a33b6bafa3');\n",
+       "var gd = document.getElementById('d488530c-df0a-4dd0-a5d4-bd55b79cf332');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -7018,10 +7018,10 @@
    "execution_count": 25,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T00:59:19.639547Z",
-     "iopub.status.busy": "2024-12-14T00:59:19.639233Z",
-     "iopub.status.idle": "2024-12-14T00:59:19.670098Z",
-     "shell.execute_reply": "2024-12-14T00:59:19.669543Z"
+     "iopub.execute_input": "2024-12-31T02:18:39.935154Z",
+     "iopub.status.busy": "2024-12-31T02:18:39.934983Z",
+     "iopub.status.idle": "2024-12-31T02:18:39.963922Z",
+     "shell.execute_reply": "2024-12-31T02:18:39.963440Z"
     }
    },
    "outputs": [
@@ -8160,9 +8160,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"53f8687c-322d-426d-8a84-d3e680bc0251\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"53f8687c-322d-426d-8a84-d3e680bc0251\")) {                    Plotly.newPlot(                        \"53f8687c-322d-426d-8a84-d3e680bc0251\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"x\":[\"\\u003c|endoftext|\\u003e_0\",\"When_1\",\" John_2\",\" and_3\",\" Mary_4\",\" went_5\",\" to_6\",\" the_7\",\" shops_8\",\",_9\",\" John_10\",\" gave_11\",\" the_12\",\" bag_13\",\" to_14\"],\"z\":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.8507904410362244,-0.00027849970501847565,-7.350136729655787e-05,-0.00047316084965132177,3.9704162190901116e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.008864667266607285,0.00022147741401568055,0.00014985552115831524,-4.8866662837099284e-05,0.000304174842312932],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.013550027273595333,5.839834557264112e-05,-0.0003300177922938019,-0.0006387235480360687,0.0007728383643552661],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0019474170403555036,0.0004988359869457781,0.00017297992599196732,0.00016838189912959933,0.0004079157952219248],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.019786769524216652,0.004128897562623024,-4.849747710977681e-05,-0.00016989219875540584,0.0007919353083707392],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.09652478247880936,-0.001882205600850284,-0.00048380010412074625,0.000710244697984308,-0.00018361916590947658],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.015899553894996643,-0.0008504678844474256,0.00012357627565506846,2.8863551051472314e-05,-0.007237736601382494],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.01036097388714552,0.003150892909616232,0.0005310557899065316,0.00023543930728919804,0.008496489375829697],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.012532956898212433,2.2285346858552657e-05,-0.0003539476892910898,8.638927829451859e-05,-0.02163148671388626],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.00033434730721637607,0.0008097903919406235,1.6344823961844668e-05,0.00012938254803884774,0.031624484807252884],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0013600102392956614,-0.0001945940311998129,-9.914294059854001e-05,-0.00014133071817923337,0.02876460924744606],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.020449254661798477]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Position: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Position\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Patched MLP Layer\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"11229a77-9a7e-45c1-802c-f409a3943559\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"11229a77-9a7e-45c1-802c-f409a3943559\")) {                    Plotly.newPlot(                        \"11229a77-9a7e-45c1-802c-f409a3943559\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"x\":[\"\\u003c|endoftext|\\u003e_0\",\"When_1\",\" John_2\",\" and_3\",\" Mary_4\",\" went_5\",\" to_6\",\" the_7\",\" shops_8\",\",_9\",\" John_10\",\" gave_11\",\" the_12\",\" bag_13\",\" to_14\"],\"z\":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.8507904410362244,-0.00027849970501847565,-7.350136729655787e-05,-0.00047316084965132177,3.9704162190901116e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.008864667266607285,0.00022147741401568055,0.00014985552115831524,-4.8866662837099284e-05,0.000304174842312932],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.013550027273595333,5.839834557264112e-05,-0.0003300177922938019,-0.0006387235480360687,0.0007728383643552661],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0019474170403555036,0.0004988359869457781,0.00017297992599196732,0.00016838189912959933,0.0004079157952219248],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.019786769524216652,0.004128897562623024,-4.849747710977681e-05,-0.00016989219875540584,0.0007919353083707392],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.09652478247880936,-0.001882205600850284,-0.00048380010412074625,0.000710244697984308,-0.00018361916590947658],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.015899553894996643,-0.0008504678844474256,0.00012357627565506846,2.8863551051472314e-05,-0.007237736601382494],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.01036097388714552,0.003150892909616232,0.0005310557899065316,0.00023543930728919804,0.008496489375829697],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.012532956898212433,2.2285346858552657e-05,-0.0003539476892910898,8.638927829451859e-05,-0.02163148671388626],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.00033434730721637607,0.0008097903919406235,1.6344823961844668e-05,0.00012938254803884774,0.031624484807252884],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0013600102392956614,-0.0001945940311998129,-9.914294059854001e-05,-0.00014133071817923337,0.02876460924744606],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.020449254661798477]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Position: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Position\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Patched MLP Layer\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('53f8687c-322d-426d-8a84-d3e680bc0251');\n",
+       "var gd = document.getElementById('11229a77-9a7e-45c1-802c-f409a3943559');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -8216,10 +8216,10 @@
    "execution_count": 26,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T00:59:19.672437Z",
-     "iopub.status.busy": "2024-12-14T00:59:19.672240Z",
-     "iopub.status.idle": "2024-12-14T00:59:57.559193Z",
-     "shell.execute_reply": "2024-12-14T00:59:57.558632Z"
+     "iopub.execute_input": "2024-12-31T02:18:39.966091Z",
+     "iopub.status.busy": "2024-12-31T02:18:39.965770Z",
+     "iopub.status.idle": "2024-12-31T02:19:16.878435Z",
+     "shell.execute_reply": "2024-12-31T02:19:16.877881Z"
     }
    },
    "outputs": [],
@@ -8266,10 +8266,10 @@
    "execution_count": 27,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T00:59:57.561760Z",
-     "iopub.status.busy": "2024-12-14T00:59:57.561408Z",
-     "iopub.status.idle": "2024-12-14T00:59:57.591996Z",
-     "shell.execute_reply": "2024-12-14T00:59:57.591441Z"
+     "iopub.execute_input": "2024-12-31T02:19:16.881037Z",
+     "iopub.status.busy": "2024-12-31T02:19:16.880691Z",
+     "iopub.status.idle": "2024-12-31T02:19:16.910779Z",
+     "shell.execute_reply": "2024-12-31T02:19:16.910243Z"
     }
    },
    "outputs": [
@@ -9355,9 +9355,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"d7db9d56-c782-4460-85c8-5165d9b90be4\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"d7db9d56-c782-4460-85c8-5165d9b90be4\")) {                    Plotly.newPlot(                        \"d7db9d56-c782-4460-85c8-5165d9b90be4\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.000949342327658087,0.016124119982123375,0.0018556242575868964,0.0034389242064207792,-0.009823138825595379,0.011059673503041267,-0.0040638200007379055,-0.0015785341383889318,-0.001208073808811605,0.003828749991953373,-0.004257441032677889,-0.0011428623693063855],[-0.001076644053682685,-0.0003779782564379275,3.0206042538338806e-06,-0.000260510336374864,-0.00014149851631373167,0.0038328110240399837,-0.0004281202855054289,-0.0014291484840214252,-0.0009217541082762182,0.0006938663427717984,0.00043359093251638114,-0.0035708239302039146],[-0.0004966544220224023,0.0008070718613453209,0.0005420306115411222,-0.0005315592279657722,-0.0007152119651436806,-0.0010380810126662254,-0.0009492752142250538,-8.538240945199504e-05,0.0002774928288999945,0.0021076432894915342,-0.0001976146304514259,-0.0016398524167016149],[0.11626167595386505,0.00025017315056174994,-0.0014669395750388503,-0.0003969745012000203,0.01896214485168457,-0.00018882131553255022,0.011170395649969578,-0.0013304082676768303,-0.0007363561890088022,-0.00030310085276141763,-0.00014579448907170445,-0.0002225178322987631],[-0.0016502231592312455,0.00029299859306775033,-0.0014352232683449984,0.030841777101159096,-0.007431894540786743,-0.0002820908557623625,0.006017110776156187,-0.011006947606801987,-0.0012651296565309167,0.001490030437707901,-0.00018016225658357143,0.0029450219590216875],[-0.004210856277495623,0.0029608632903546095,0.002045620232820511,0.0013410139363259077,-0.0012189144035801291,0.34349939227104187,0.0005627049831673503,-0.0001257913827430457,-0.005152345169335604,0.016241183504462242,0.017089638859033585,-0.004173669498413801],[0.039775215089321136,0.015227536670863628,-0.001023011514917016,0.0008081123232841492,-0.004935599863529205,-0.002123551908880472,-0.014273260720074177,0.0013749118661507964,0.0014838549541309476,0.1302708089351654,-0.00033538774005137384,0.0012914760736748576],[0.0003724069392774254,0.0195147804915905,0.00022308839834295213,0.12424899637699127,-0.0004041232750751078,-0.007652163505554199,0.0013012762647122145,-0.0011244030902162194,-0.007449581753462553,0.19224178791046143,-0.0032753082923591137,-0.000501957256346941],[-0.001007740618661046,3.1212908652378246e-05,-0.0008586570620536804,0.012360110878944397,-0.00040429108776152134,-0.004328995477408171,0.31855517625808716,0.002329523442313075,0.002118081087246537,0.0001410957775078714,0.27793484926223755,0.005738543812185526],[0.0058897752314805984,-0.000968372100032866,0.009125681594014168,0.020675498992204666,-0.03700448200106621,0.014263628050684929,-0.04828385263681412,0.05834108963608742,0.0006516450084745884,0.2636081576347351,0.0004918886115774512,-0.0026110773906111717],[0.08374276012182236,0.020676370710134506,-0.0037437702994793653,0.01085111778229475,-0.0010960765648633242,0.0004738992138300091,0.04817944020032883,-0.47990989685058594,0.00018412260396871716,0.011861777864396572,0.06088537722826004,0.0008459034143015742],[0.005328110884875059,-0.011493667028844357,-0.11350765824317932,0.006330112461000681,0.0003166264505125582,-0.0011600126745179296,-0.022669030353426933,0.004070901777595282,0.007316238712519407,-0.008345123380422592,-0.27817434072494507,0.003635196480900049]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Patched Head Output\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"c9735339-8c98-45e1-988a-a5a0017e244e\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"c9735339-8c98-45e1-988a-a5a0017e244e\")) {                    Plotly.newPlot(                        \"c9735339-8c98-45e1-988a-a5a0017e244e\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.000949342327658087,0.016124119982123375,0.0018556242575868964,0.0034389242064207792,-0.009823138825595379,0.011059673503041267,-0.0040638200007379055,-0.0015785341383889318,-0.001208073808811605,0.003828749991953373,-0.004257441032677889,-0.0011428623693063855],[-0.001076644053682685,-0.0003779782564379275,3.0206042538338806e-06,-0.000260510336374864,-0.00014149851631373167,0.0038328110240399837,-0.0004281202855054289,-0.0014291484840214252,-0.0009217541082762182,0.0006938663427717984,0.00043359093251638114,-0.0035708239302039146],[-0.0004966544220224023,0.0008070718613453209,0.0005420306115411222,-0.0005315592279657722,-0.0007152119651436806,-0.0010380810126662254,-0.0009492752142250538,-8.538240945199504e-05,0.0002774928288999945,0.0021076432894915342,-0.0001976146304514259,-0.0016398524167016149],[0.11626167595386505,0.00025017315056174994,-0.0014669395750388503,-0.0003969745012000203,0.01896214485168457,-0.00018882131553255022,0.011170395649969578,-0.0013304082676768303,-0.0007363561890088022,-0.00030310085276141763,-0.00014579448907170445,-0.0002225178322987631],[-0.0016502231592312455,0.00029299859306775033,-0.0014352232683449984,0.030841777101159096,-0.007431894540786743,-0.0002820908557623625,0.006017110776156187,-0.011006947606801987,-0.0012651296565309167,0.001490030437707901,-0.00018016225658357143,0.0029450219590216875],[-0.004210856277495623,0.0029608632903546095,0.002045620232820511,0.0013410139363259077,-0.0012189144035801291,0.34349939227104187,0.0005627049831673503,-0.0001257913827430457,-0.005152345169335604,0.016241183504462242,0.017089638859033585,-0.004173669498413801],[0.039775215089321136,0.015227536670863628,-0.001023011514917016,0.0008081123232841492,-0.004935599863529205,-0.002123551908880472,-0.014273260720074177,0.0013749118661507964,0.0014838549541309476,0.1302708089351654,-0.00033538774005137384,0.0012914760736748576],[0.0003724069392774254,0.0195147804915905,0.00022308839834295213,0.12424899637699127,-0.0004041232750751078,-0.007652163505554199,0.0013012762647122145,-0.0011244030902162194,-0.007449581753462553,0.19224178791046143,-0.0032753082923591137,-0.000501957256346941],[-0.001007740618661046,3.1212908652378246e-05,-0.0008586570620536804,0.012360110878944397,-0.00040429108776152134,-0.004328995477408171,0.31855517625808716,0.002329523442313075,0.002118081087246537,0.0001410957775078714,0.27793484926223755,0.005738543812185526],[0.0058897752314805984,-0.000968372100032866,0.009125681594014168,0.020675498992204666,-0.03700448200106621,0.014263628050684929,-0.04828385263681412,0.05834108963608742,0.0006516450084745884,0.2636081576347351,0.0004918886115774512,-0.0026110773906111717],[0.08374276012182236,0.020676370710134506,-0.0037437702994793653,0.01085111778229475,-0.0010960765648633242,0.0004738992138300091,0.04817944020032883,-0.47990989685058594,0.00018412260396871716,0.011861777864396572,0.06088537722826004,0.0008459034143015742],[0.005328110884875059,-0.011493667028844357,-0.11350765824317932,0.006330112461000681,0.0003166264505125582,-0.0011600126745179296,-0.022669030353426933,0.004070901777595282,0.007316238712519407,-0.008345123380422592,-0.27817434072494507,0.003635196480900049]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Patched Head Output\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('d7db9d56-c782-4460-85c8-5165d9b90be4');\n",
+       "var gd = document.getElementById('c9735339-8c98-45e1-988a-a5a0017e244e');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -9420,10 +9420,10 @@
    "execution_count": 28,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T00:59:57.594318Z",
-     "iopub.status.busy": "2024-12-14T00:59:57.594131Z",
-     "iopub.status.idle": "2024-12-14T01:00:35.562582Z",
-     "shell.execute_reply": "2024-12-14T01:00:35.562008Z"
+     "iopub.execute_input": "2024-12-31T02:19:16.913031Z",
+     "iopub.status.busy": "2024-12-31T02:19:16.912858Z",
+     "iopub.status.idle": "2024-12-31T02:19:53.859146Z",
+     "shell.execute_reply": "2024-12-31T02:19:53.858421Z"
     }
    },
    "outputs": [],
@@ -9458,10 +9458,10 @@
    "execution_count": 29,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:00:35.565275Z",
-     "iopub.status.busy": "2024-12-14T01:00:35.564875Z",
-     "iopub.status.idle": "2024-12-14T01:00:35.595964Z",
-     "shell.execute_reply": "2024-12-14T01:00:35.595316Z"
+     "iopub.execute_input": "2024-12-31T02:19:53.861763Z",
+     "iopub.status.busy": "2024-12-31T02:19:53.861588Z",
+     "iopub.status.idle": "2024-12-31T02:19:53.891463Z",
+     "shell.execute_reply": "2024-12-31T02:19:53.890885Z"
     }
    },
    "outputs": [
@@ -10547,9 +10547,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"b78bb017-f4f8-4418-b905-eac4d9764ea8\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"b78bb017-f4f8-4418-b905-eac4d9764ea8\")) {                    Plotly.newPlot(                        \"b78bb017-f4f8-4418-b905-eac4d9764ea8\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[-0.00019808451179414988,0.005339857656508684,0.0006520813331007957,0.0035054110921919346,-0.008983243256807327,0.003481481224298477,-0.0008612077799625695,-3.356226807227358e-05,0.0005169931682758033,0.000442283577285707,-0.003907017409801483,-0.0001884857047116384],[-0.0004399341996759176,-0.00044604254071600735,-6.719166412949562e-05,7.296437252080068e-05,-3.587806349969469e-05,-0.001932314014993608,-0.0001571049797348678,1.550576780573465e-05,0.00020597163529600948,0.0003376699751242995,0.00035139694227837026,-0.0005663632764481008],[0.0002103011793224141,-0.0007198435487225652,0.00048675358993932605,-0.0005971734644845128,-0.0005919041577726603,-0.0005438430234789848,-0.0002275186125189066,-0.00047960481606423855,0.00020617301925085485,0.0011842446401715279,-0.00035743816988542676,-0.0009093025582842529],[0.0010397254955023527,-0.0001205221051350236,-7.69247199059464e-05,-0.0007269251509569585,-0.001310002407990396,-0.002310896525159478,0.010987145826220512,-5.06454634887632e-05,0.00014364650996867567,0.00015129870735108852,-7.920695497887209e-05,-1.9667490050778724e-05],[-0.0005367277772165835,-0.0008130795322358608,-0.000133275767439045,0.030610064044594765,-0.007185581140220165,0.00014941922563593835,0.001333831693045795,-0.011422313749790192,-0.0005329688428901136,0.0005127307958900928,0.00037368229823186994,0.002956030424684286],[7.4508234320092015e-06,6.712453796353657e-06,0.0015982352197170258,0.00033918028930202127,-0.0012586521916091442,-5.423662514658645e-05,0.0006328836898319423,-0.0002699077595025301,7.430685946019366e-05,-0.006704063154757023,0.0031767694745212793,-0.001728456816636026],[0.04863159358501434,0.015315738506615162,-0.0004640990518964827,-0.0001160583269665949,-4.839679240831174e-05,-0.003953501116484404,-0.017375119030475616,-0.0001535809424240142,0.0012203240767121315,-0.00017975950322579592,-0.0004269791825208813,0.0001233748917002231],[-2.849436532414984e-05,-0.001384309260174632,-0.00012176390737295151,0.13321569561958313,-0.00024480317370034754,-0.0073145609349012375,0.0003329376922920346,-0.000794418912846595,-0.007937678135931492,0.20841389894485474,-0.0001913720479933545,-0.0002066764427581802],[-0.0020480367820709944,-0.0003764008288271725,-0.0033146096393465996,-0.009665866382420063,-0.00031719700200483203,-0.005141303408890963,0.3171706795692444,0.002842522691935301,0.00047299303696490824,-0.0011535687372088432,0.27267059683799744,-0.00317572895437479],[-0.00043966571683995426,5.7257228036178276e-05,-0.002063408261165023,0.020066477358341217,-0.007870486006140709,0.01131565310060978,0.003057321300730109,0.06856381893157959,-0.0027472060173749924,-0.009278557263314724,0.0005067567108199,-0.0013152381870895624],[-0.012957250699400902,-0.00304490327835083,-0.017922621220350266,-0.004358463454991579,-0.0011523604625836015,0.0004998764488846064,-0.0031132360454648733,0.01958620175719261,4.3899446609430015e-05,0.012970171868801117,-0.007695660460740328,-0.0009143368806689978],[0.004100503865629435,-0.020459022372961044,-0.035875849425792694,0.014655501581728458,0.0008445944986306131,0.0017805789830163121,-0.018042270094156265,0.003519205143675208,0.008252425119280815,-0.0017664493061602116,0.044166937470436096,0.006474765483289957]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Patched Head Value\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"b42e697c-5002-48bc-8c51-ecb852488511\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"b42e697c-5002-48bc-8c51-ecb852488511\")) {                    Plotly.newPlot(                        \"b42e697c-5002-48bc-8c51-ecb852488511\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[-0.00019808451179414988,0.005339857656508684,0.0006520813331007957,0.0035054110921919346,-0.008983243256807327,0.003481481224298477,-0.0008612077799625695,-3.356226807227358e-05,0.0005169931682758033,0.000442283577285707,-0.003907017409801483,-0.0001884857047116384],[-0.0004399341996759176,-0.00044604254071600735,-6.719166412949562e-05,7.296437252080068e-05,-3.587806349969469e-05,-0.001932314014993608,-0.0001571049797348678,1.550576780573465e-05,0.00020597163529600948,0.0003376699751242995,0.00035139694227837026,-0.0005663632764481008],[0.0002103011793224141,-0.0007198435487225652,0.00048675358993932605,-0.0005971734644845128,-0.0005919041577726603,-0.0005438430234789848,-0.0002275186125189066,-0.00047960481606423855,0.00020617301925085485,0.0011842446401715279,-0.00035743816988542676,-0.0009093025582842529],[0.0010397254955023527,-0.0001205221051350236,-7.69247199059464e-05,-0.0007269251509569585,-0.001310002407990396,-0.002310896525159478,0.010987145826220512,-5.06454634887632e-05,0.00014364650996867567,0.00015129870735108852,-7.920695497887209e-05,-1.9667490050778724e-05],[-0.0005367277772165835,-0.0008130795322358608,-0.000133275767439045,0.030610064044594765,-0.007185581140220165,0.00014941922563593835,0.001333831693045795,-0.011422313749790192,-0.0005329688428901136,0.0005127307958900928,0.00037368229823186994,0.002956030424684286],[7.4508234320092015e-06,6.712453796353657e-06,0.0015982352197170258,0.00033918028930202127,-0.0012586521916091442,-5.423662514658645e-05,0.0006328836898319423,-0.0002699077595025301,7.430685946019366e-05,-0.006704063154757023,0.0031767694745212793,-0.001728456816636026],[0.04863159358501434,0.015315738506615162,-0.0004640990518964827,-0.0001160583269665949,-4.839679240831174e-05,-0.003953501116484404,-0.017375119030475616,-0.0001535809424240142,0.0012203240767121315,-0.00017975950322579592,-0.0004269791825208813,0.0001233748917002231],[-2.849436532414984e-05,-0.001384309260174632,-0.00012176390737295151,0.13321569561958313,-0.00024480317370034754,-0.0073145609349012375,0.0003329376922920346,-0.000794418912846595,-0.007937678135931492,0.20841389894485474,-0.0001913720479933545,-0.0002066764427581802],[-0.0020480367820709944,-0.0003764008288271725,-0.0033146096393465996,-0.009665866382420063,-0.00031719700200483203,-0.005141303408890963,0.3171706795692444,0.002842522691935301,0.00047299303696490824,-0.0011535687372088432,0.27267059683799744,-0.00317572895437479],[-0.00043966571683995426,5.7257228036178276e-05,-0.002063408261165023,0.020066477358341217,-0.007870486006140709,0.01131565310060978,0.003057321300730109,0.06856381893157959,-0.0027472060173749924,-0.009278557263314724,0.0005067567108199,-0.0013152381870895624],[-0.012957250699400902,-0.00304490327835083,-0.017922621220350266,-0.004358463454991579,-0.0011523604625836015,0.0004998764488846064,-0.0031132360454648733,0.01958620175719261,4.3899446609430015e-05,0.012970171868801117,-0.007695660460740328,-0.0009143368806689978],[0.004100503865629435,-0.020459022372961044,-0.035875849425792694,0.014655501581728458,0.0008445944986306131,0.0017805789830163121,-0.018042270094156265,0.003519205143675208,0.008252425119280815,-0.0017664493061602116,0.044166937470436096,0.006474765483289957]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Patched Head Value\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('b78bb017-f4f8-4418-b905-eac4d9764ea8');\n",
+       "var gd = document.getElementById('b42e697c-5002-48bc-8c51-ecb852488511');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -10600,10 +10600,10 @@
    "execution_count": 30,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:00:35.598084Z",
-     "iopub.status.busy": "2024-12-14T01:00:35.597908Z",
-     "iopub.status.idle": "2024-12-14T01:00:35.643804Z",
-     "shell.execute_reply": "2024-12-14T01:00:35.643178Z"
+     "iopub.execute_input": "2024-12-31T02:19:53.893671Z",
+     "iopub.status.busy": "2024-12-31T02:19:53.893271Z",
+     "iopub.status.idle": "2024-12-31T02:19:53.937319Z",
+     "shell.execute_reply": "2024-12-31T02:19:53.936756Z"
     }
    },
    "outputs": [
@@ -12117,9 +12117,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"c2250b6e-3006-4a68-a458-2890b3c0d4df\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"c2250b6e-3006-4a68-a458-2890b3c0d4df\")) {                    Plotly.newPlot(                        \"c2250b6e-3006-4a68-a458-2890b3c0d4df\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003eValue Patch=%{x}\\u003cbr\\u003eOutput Patch=%{y}\\u003cbr\\u003eLayer=%{marker.color}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"L0H0\",\"L0H1\",\"L0H2\",\"L0H3\",\"L0H4\",\"L0H5\",\"L0H6\",\"L0H7\",\"L0H8\",\"L0H9\",\"L0H10\",\"L0H11\",\"L1H0\",\"L1H1\",\"L1H2\",\"L1H3\",\"L1H4\",\"L1H5\",\"L1H6\",\"L1H7\",\"L1H8\",\"L1H9\",\"L1H10\",\"L1H11\",\"L2H0\",\"L2H1\",\"L2H2\",\"L2H3\",\"L2H4\",\"L2H5\",\"L2H6\",\"L2H7\",\"L2H8\",\"L2H9\",\"L2H10\",\"L2H11\",\"L3H0\",\"L3H1\",\"L3H2\",\"L3H3\",\"L3H4\",\"L3H5\",\"L3H6\",\"L3H7\",\"L3H8\",\"L3H9\",\"L3H10\",\"L3H11\",\"L4H0\",\"L4H1\",\"L4H2\",\"L4H3\",\"L4H4\",\"L4H5\",\"L4H6\",\"L4H7\",\"L4H8\",\"L4H9\",\"L4H10\",\"L4H11\",\"L5H0\",\"L5H1\",\"L5H2\",\"L5H3\",\"L5H4\",\"L5H5\",\"L5H6\",\"L5H7\",\"L5H8\",\"L5H9\",\"L5H10\",\"L5H11\",\"L6H0\",\"L6H1\",\"L6H2\",\"L6H3\",\"L6H4\",\"L6H5\",\"L6H6\",\"L6H7\",\"L6H8\",\"L6H9\",\"L6H10\",\"L6H11\",\"L7H0\",\"L7H1\",\"L7H2\",\"L7H3\",\"L7H4\",\"L7H5\",\"L7H6\",\"L7H7\",\"L7H8\",\"L7H9\",\"L7H10\",\"L7H11\",\"L8H0\",\"L8H1\",\"L8H2\",\"L8H3\",\"L8H4\",\"L8H5\",\"L8H6\",\"L8H7\",\"L8H8\",\"L8H9\",\"L8H10\",\"L8H11\",\"L9H0\",\"L9H1\",\"L9H2\",\"L9H3\",\"L9H4\",\"L9H5\",\"L9H6\",\"L9H7\",\"L9H8\",\"L9H9\",\"L9H10\",\"L9H11\",\"L10H0\",\"L10H1\",\"L10H2\",\"L10H3\",\"L10H4\",\"L10H5\",\"L10H6\",\"L10H7\",\"L10H8\",\"L10H9\",\"L10H10\",\"L10H11\",\"L11H0\",\"L11H1\",\"L11H2\",\"L11H3\",\"L11H4\",\"L11H5\",\"L11H6\",\"L11H7\",\"L11H8\",\"L11H9\",\"L11H10\",\"L11H11\"],\"legendgroup\":\"\",\"marker\":{\"color\":[0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7,7,7,7,7,8,8,8,8,8,8,8,8,8,8,8,8,9,9,9,9,9,9,9,9,9,9,9,9,10,10,10,10,10,10,10,10,10,10,10,10,11,11,11,11,11,11,11,11,11,11,11,11],\"coloraxis\":\"coloraxis\",\"symbol\":\"circle\"},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[-0.00019808451179414988,0.005339857656508684,0.0006520813331007957,0.0035054110921919346,-0.008983243256807327,0.003481481224298477,-0.0008612077799625695,-3.356226807227358e-05,0.0005169931682758033,0.000442283577285707,-0.003907017409801483,-0.0001884857047116384,-0.0004399341996759176,-0.00044604254071600735,-6.719166412949562e-05,7.296437252080068e-05,-3.587806349969469e-05,-0.001932314014993608,-0.0001571049797348678,1.550576780573465e-05,0.00020597163529600948,0.0003376699751242995,0.00035139694227837026,-0.0005663632764481008,0.0002103011793224141,-0.0007198435487225652,0.00048675358993932605,-0.0005971734644845128,-0.0005919041577726603,-0.0005438430234789848,-0.0002275186125189066,-0.00047960481606423855,0.00020617301925085485,0.0011842446401715279,-0.00035743816988542676,-0.0009093025582842529,0.0010397254955023527,-0.0001205221051350236,-7.69247199059464e-05,-0.0007269251509569585,-0.001310002407990396,-0.002310896525159478,0.010987145826220512,-5.06454634887632e-05,0.00014364650996867567,0.00015129870735108852,-7.920695497887209e-05,-1.9667490050778724e-05,-0.0005367277772165835,-0.0008130795322358608,-0.000133275767439045,0.030610064044594765,-0.007185581140220165,0.00014941922563593835,0.001333831693045795,-0.011422313749790192,-0.0005329688428901136,0.0005127307958900928,0.00037368229823186994,0.002956030424684286,7.4508234320092015e-06,6.712453796353657e-06,0.0015982352197170258,0.00033918028930202127,-0.0012586521916091442,-5.423662514658645e-05,0.0006328836898319423,-0.0002699077595025301,7.430685946019366e-05,-0.006704063154757023,0.0031767694745212793,-0.001728456816636026,0.04863159358501434,0.015315738506615162,-0.0004640990518964827,-0.0001160583269665949,-4.839679240831174e-05,-0.003953501116484404,-0.017375119030475616,-0.0001535809424240142,0.0012203240767121315,-0.00017975950322579592,-0.0004269791825208813,0.0001233748917002231,-2.849436532414984e-05,-0.001384309260174632,-0.00012176390737295151,0.13321569561958313,-0.00024480317370034754,-0.0073145609349012375,0.0003329376922920346,-0.000794418912846595,-0.007937678135931492,0.20841389894485474,-0.0001913720479933545,-0.0002066764427581802,-0.0020480367820709944,-0.0003764008288271725,-0.0033146096393465996,-0.009665866382420063,-0.00031719700200483203,-0.005141303408890963,0.3171706795692444,0.002842522691935301,0.00047299303696490824,-0.0011535687372088432,0.27267059683799744,-0.00317572895437479,-0.00043966571683995426,5.7257228036178276e-05,-0.002063408261165023,0.020066477358341217,-0.007870486006140709,0.01131565310060978,0.003057321300730109,0.06856381893157959,-0.0027472060173749924,-0.009278557263314724,0.0005067567108199,-0.0013152381870895624,-0.012957250699400902,-0.00304490327835083,-0.017922621220350266,-0.004358463454991579,-0.0011523604625836015,0.0004998764488846064,-0.0031132360454648733,0.01958620175719261,4.3899446609430015e-05,0.012970171868801117,-0.007695660460740328,-0.0009143368806689978,0.004100503865629435,-0.020459022372961044,-0.035875849425792694,0.014655501581728458,0.0008445944986306131,0.0017805789830163121,-0.018042270094156265,0.003519205143675208,0.008252425119280815,-0.0017664493061602116,0.044166937470436096,0.006474765483289957],\"xaxis\":\"x\",\"y\":[0.000949342327658087,0.016124119982123375,0.0018556242575868964,0.0034389242064207792,-0.009823138825595379,0.011059673503041267,-0.0040638200007379055,-0.0015785341383889318,-0.001208073808811605,0.003828749991953373,-0.004257441032677889,-0.0011428623693063855,-0.001076644053682685,-0.0003779782564379275,3.0206042538338806e-06,-0.000260510336374864,-0.00014149851631373167,0.0038328110240399837,-0.0004281202855054289,-0.0014291484840214252,-0.0009217541082762182,0.0006938663427717984,0.00043359093251638114,-0.0035708239302039146,-0.0004966544220224023,0.0008070718613453209,0.0005420306115411222,-0.0005315592279657722,-0.0007152119651436806,-0.0010380810126662254,-0.0009492752142250538,-8.538240945199504e-05,0.0002774928288999945,0.0021076432894915342,-0.0001976146304514259,-0.0016398524167016149,0.11626167595386505,0.00025017315056174994,-0.0014669395750388503,-0.0003969745012000203,0.01896214485168457,-0.00018882131553255022,0.011170395649969578,-0.0013304082676768303,-0.0007363561890088022,-0.00030310085276141763,-0.00014579448907170445,-0.0002225178322987631,-0.0016502231592312455,0.00029299859306775033,-0.0014352232683449984,0.030841777101159096,-0.007431894540786743,-0.0002820908557623625,0.006017110776156187,-0.011006947606801987,-0.0012651296565309167,0.001490030437707901,-0.00018016225658357143,0.0029450219590216875,-0.004210856277495623,0.0029608632903546095,0.002045620232820511,0.0013410139363259077,-0.0012189144035801291,0.34349939227104187,0.0005627049831673503,-0.0001257913827430457,-0.005152345169335604,0.016241183504462242,0.017089638859033585,-0.004173669498413801,0.039775215089321136,0.015227536670863628,-0.001023011514917016,0.0008081123232841492,-0.004935599863529205,-0.002123551908880472,-0.014273260720074177,0.0013749118661507964,0.0014838549541309476,0.1302708089351654,-0.00033538774005137384,0.0012914760736748576,0.0003724069392774254,0.0195147804915905,0.00022308839834295213,0.12424899637699127,-0.0004041232750751078,-0.007652163505554199,0.0013012762647122145,-0.0011244030902162194,-0.007449581753462553,0.19224178791046143,-0.0032753082923591137,-0.000501957256346941,-0.001007740618661046,3.1212908652378246e-05,-0.0008586570620536804,0.012360110878944397,-0.00040429108776152134,-0.004328995477408171,0.31855517625808716,0.002329523442313075,0.002118081087246537,0.0001410957775078714,0.27793484926223755,0.005738543812185526,0.0058897752314805984,-0.000968372100032866,0.009125681594014168,0.020675498992204666,-0.03700448200106621,0.014263628050684929,-0.04828385263681412,0.05834108963608742,0.0006516450084745884,0.2636081576347351,0.0004918886115774512,-0.0026110773906111717,0.08374276012182236,0.020676370710134506,-0.0037437702994793653,0.01085111778229475,-0.0010960765648633242,0.0004738992138300091,0.04817944020032883,-0.47990989685058594,0.00018412260396871716,0.011861777864396572,0.06088537722826004,0.0008459034143015742,0.005328110884875059,-0.011493667028844357,-0.11350765824317932,0.006330112461000681,0.0003166264505125582,-0.0011600126745179296,-0.022669030353426933,0.004070901777595282,0.007316238712519407,-0.008345123380422592,-0.27817434072494507,0.003635196480900049],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Value Patch\"},\"range\":[-0.5,0.5]},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Output Patch\"},\"range\":[-0.5,0.5]},\"coloraxis\":{\"colorbar\":{\"title\":{\"text\":\"Layer\"}},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Scatter plot of output patching vs value patching\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"40325163-a9c5-4067-9615-e2cea91c5832\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"40325163-a9c5-4067-9615-e2cea91c5832\")) {                    Plotly.newPlot(                        \"40325163-a9c5-4067-9615-e2cea91c5832\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003eValue Patch=%{x}\\u003cbr\\u003eOutput Patch=%{y}\\u003cbr\\u003eLayer=%{marker.color}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"L0H0\",\"L0H1\",\"L0H2\",\"L0H3\",\"L0H4\",\"L0H5\",\"L0H6\",\"L0H7\",\"L0H8\",\"L0H9\",\"L0H10\",\"L0H11\",\"L1H0\",\"L1H1\",\"L1H2\",\"L1H3\",\"L1H4\",\"L1H5\",\"L1H6\",\"L1H7\",\"L1H8\",\"L1H9\",\"L1H10\",\"L1H11\",\"L2H0\",\"L2H1\",\"L2H2\",\"L2H3\",\"L2H4\",\"L2H5\",\"L2H6\",\"L2H7\",\"L2H8\",\"L2H9\",\"L2H10\",\"L2H11\",\"L3H0\",\"L3H1\",\"L3H2\",\"L3H3\",\"L3H4\",\"L3H5\",\"L3H6\",\"L3H7\",\"L3H8\",\"L3H9\",\"L3H10\",\"L3H11\",\"L4H0\",\"L4H1\",\"L4H2\",\"L4H3\",\"L4H4\",\"L4H5\",\"L4H6\",\"L4H7\",\"L4H8\",\"L4H9\",\"L4H10\",\"L4H11\",\"L5H0\",\"L5H1\",\"L5H2\",\"L5H3\",\"L5H4\",\"L5H5\",\"L5H6\",\"L5H7\",\"L5H8\",\"L5H9\",\"L5H10\",\"L5H11\",\"L6H0\",\"L6H1\",\"L6H2\",\"L6H3\",\"L6H4\",\"L6H5\",\"L6H6\",\"L6H7\",\"L6H8\",\"L6H9\",\"L6H10\",\"L6H11\",\"L7H0\",\"L7H1\",\"L7H2\",\"L7H3\",\"L7H4\",\"L7H5\",\"L7H6\",\"L7H7\",\"L7H8\",\"L7H9\",\"L7H10\",\"L7H11\",\"L8H0\",\"L8H1\",\"L8H2\",\"L8H3\",\"L8H4\",\"L8H5\",\"L8H6\",\"L8H7\",\"L8H8\",\"L8H9\",\"L8H10\",\"L8H11\",\"L9H0\",\"L9H1\",\"L9H2\",\"L9H3\",\"L9H4\",\"L9H5\",\"L9H6\",\"L9H7\",\"L9H8\",\"L9H9\",\"L9H10\",\"L9H11\",\"L10H0\",\"L10H1\",\"L10H2\",\"L10H3\",\"L10H4\",\"L10H5\",\"L10H6\",\"L10H7\",\"L10H8\",\"L10H9\",\"L10H10\",\"L10H11\",\"L11H0\",\"L11H1\",\"L11H2\",\"L11H3\",\"L11H4\",\"L11H5\",\"L11H6\",\"L11H7\",\"L11H8\",\"L11H9\",\"L11H10\",\"L11H11\"],\"legendgroup\":\"\",\"marker\":{\"color\":[0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7,7,7,7,7,8,8,8,8,8,8,8,8,8,8,8,8,9,9,9,9,9,9,9,9,9,9,9,9,10,10,10,10,10,10,10,10,10,10,10,10,11,11,11,11,11,11,11,11,11,11,11,11],\"coloraxis\":\"coloraxis\",\"symbol\":\"circle\"},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[-0.00019808451179414988,0.005339857656508684,0.0006520813331007957,0.0035054110921919346,-0.008983243256807327,0.003481481224298477,-0.0008612077799625695,-3.356226807227358e-05,0.0005169931682758033,0.000442283577285707,-0.003907017409801483,-0.0001884857047116384,-0.0004399341996759176,-0.00044604254071600735,-6.719166412949562e-05,7.296437252080068e-05,-3.587806349969469e-05,-0.001932314014993608,-0.0001571049797348678,1.550576780573465e-05,0.00020597163529600948,0.0003376699751242995,0.00035139694227837026,-0.0005663632764481008,0.0002103011793224141,-0.0007198435487225652,0.00048675358993932605,-0.0005971734644845128,-0.0005919041577726603,-0.0005438430234789848,-0.0002275186125189066,-0.00047960481606423855,0.00020617301925085485,0.0011842446401715279,-0.00035743816988542676,-0.0009093025582842529,0.0010397254955023527,-0.0001205221051350236,-7.69247199059464e-05,-0.0007269251509569585,-0.001310002407990396,-0.002310896525159478,0.010987145826220512,-5.06454634887632e-05,0.00014364650996867567,0.00015129870735108852,-7.920695497887209e-05,-1.9667490050778724e-05,-0.0005367277772165835,-0.0008130795322358608,-0.000133275767439045,0.030610064044594765,-0.007185581140220165,0.00014941922563593835,0.001333831693045795,-0.011422313749790192,-0.0005329688428901136,0.0005127307958900928,0.00037368229823186994,0.002956030424684286,7.4508234320092015e-06,6.712453796353657e-06,0.0015982352197170258,0.00033918028930202127,-0.0012586521916091442,-5.423662514658645e-05,0.0006328836898319423,-0.0002699077595025301,7.430685946019366e-05,-0.006704063154757023,0.0031767694745212793,-0.001728456816636026,0.04863159358501434,0.015315738506615162,-0.0004640990518964827,-0.0001160583269665949,-4.839679240831174e-05,-0.003953501116484404,-0.017375119030475616,-0.0001535809424240142,0.0012203240767121315,-0.00017975950322579592,-0.0004269791825208813,0.0001233748917002231,-2.849436532414984e-05,-0.001384309260174632,-0.00012176390737295151,0.13321569561958313,-0.00024480317370034754,-0.0073145609349012375,0.0003329376922920346,-0.000794418912846595,-0.007937678135931492,0.20841389894485474,-0.0001913720479933545,-0.0002066764427581802,-0.0020480367820709944,-0.0003764008288271725,-0.0033146096393465996,-0.009665866382420063,-0.00031719700200483203,-0.005141303408890963,0.3171706795692444,0.002842522691935301,0.00047299303696490824,-0.0011535687372088432,0.27267059683799744,-0.00317572895437479,-0.00043966571683995426,5.7257228036178276e-05,-0.002063408261165023,0.020066477358341217,-0.007870486006140709,0.01131565310060978,0.003057321300730109,0.06856381893157959,-0.0027472060173749924,-0.009278557263314724,0.0005067567108199,-0.0013152381870895624,-0.012957250699400902,-0.00304490327835083,-0.017922621220350266,-0.004358463454991579,-0.0011523604625836015,0.0004998764488846064,-0.0031132360454648733,0.01958620175719261,4.3899446609430015e-05,0.012970171868801117,-0.007695660460740328,-0.0009143368806689978,0.004100503865629435,-0.020459022372961044,-0.035875849425792694,0.014655501581728458,0.0008445944986306131,0.0017805789830163121,-0.018042270094156265,0.003519205143675208,0.008252425119280815,-0.0017664493061602116,0.044166937470436096,0.006474765483289957],\"xaxis\":\"x\",\"y\":[0.000949342327658087,0.016124119982123375,0.0018556242575868964,0.0034389242064207792,-0.009823138825595379,0.011059673503041267,-0.0040638200007379055,-0.0015785341383889318,-0.001208073808811605,0.003828749991953373,-0.004257441032677889,-0.0011428623693063855,-0.001076644053682685,-0.0003779782564379275,3.0206042538338806e-06,-0.000260510336374864,-0.00014149851631373167,0.0038328110240399837,-0.0004281202855054289,-0.0014291484840214252,-0.0009217541082762182,0.0006938663427717984,0.00043359093251638114,-0.0035708239302039146,-0.0004966544220224023,0.0008070718613453209,0.0005420306115411222,-0.0005315592279657722,-0.0007152119651436806,-0.0010380810126662254,-0.0009492752142250538,-8.538240945199504e-05,0.0002774928288999945,0.0021076432894915342,-0.0001976146304514259,-0.0016398524167016149,0.11626167595386505,0.00025017315056174994,-0.0014669395750388503,-0.0003969745012000203,0.01896214485168457,-0.00018882131553255022,0.011170395649969578,-0.0013304082676768303,-0.0007363561890088022,-0.00030310085276141763,-0.00014579448907170445,-0.0002225178322987631,-0.0016502231592312455,0.00029299859306775033,-0.0014352232683449984,0.030841777101159096,-0.007431894540786743,-0.0002820908557623625,0.006017110776156187,-0.011006947606801987,-0.0012651296565309167,0.001490030437707901,-0.00018016225658357143,0.0029450219590216875,-0.004210856277495623,0.0029608632903546095,0.002045620232820511,0.0013410139363259077,-0.0012189144035801291,0.34349939227104187,0.0005627049831673503,-0.0001257913827430457,-0.005152345169335604,0.016241183504462242,0.017089638859033585,-0.004173669498413801,0.039775215089321136,0.015227536670863628,-0.001023011514917016,0.0008081123232841492,-0.004935599863529205,-0.002123551908880472,-0.014273260720074177,0.0013749118661507964,0.0014838549541309476,0.1302708089351654,-0.00033538774005137384,0.0012914760736748576,0.0003724069392774254,0.0195147804915905,0.00022308839834295213,0.12424899637699127,-0.0004041232750751078,-0.007652163505554199,0.0013012762647122145,-0.0011244030902162194,-0.007449581753462553,0.19224178791046143,-0.0032753082923591137,-0.000501957256346941,-0.001007740618661046,3.1212908652378246e-05,-0.0008586570620536804,0.012360110878944397,-0.00040429108776152134,-0.004328995477408171,0.31855517625808716,0.002329523442313075,0.002118081087246537,0.0001410957775078714,0.27793484926223755,0.005738543812185526,0.0058897752314805984,-0.000968372100032866,0.009125681594014168,0.020675498992204666,-0.03700448200106621,0.014263628050684929,-0.04828385263681412,0.05834108963608742,0.0006516450084745884,0.2636081576347351,0.0004918886115774512,-0.0026110773906111717,0.08374276012182236,0.020676370710134506,-0.0037437702994793653,0.01085111778229475,-0.0010960765648633242,0.0004738992138300091,0.04817944020032883,-0.47990989685058594,0.00018412260396871716,0.011861777864396572,0.06088537722826004,0.0008459034143015742,0.005328110884875059,-0.011493667028844357,-0.11350765824317932,0.006330112461000681,0.0003166264505125582,-0.0011600126745179296,-0.022669030353426933,0.004070901777595282,0.007316238712519407,-0.008345123380422592,-0.27817434072494507,0.003635196480900049],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Value Patch\"},\"range\":[-0.5,0.5]},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Output Patch\"},\"range\":[-0.5,0.5]},\"coloraxis\":{\"colorbar\":{\"title\":{\"text\":\"Layer\"}},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Scatter plot of output patching vs value patching\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('c2250b6e-3006-4a68-a458-2890b3c0d4df');\n",
+       "var gd = document.getElementById('40325163-a9c5-4067-9615-e2cea91c5832');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -12181,10 +12181,10 @@
    "execution_count": 31,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:00:35.646202Z",
-     "iopub.status.busy": "2024-12-14T01:00:35.645977Z",
-     "iopub.status.idle": "2024-12-14T01:01:13.540641Z",
-     "shell.execute_reply": "2024-12-14T01:01:13.539993Z"
+     "iopub.execute_input": "2024-12-31T02:19:53.939836Z",
+     "iopub.status.busy": "2024-12-31T02:19:53.939510Z",
+     "iopub.status.idle": "2024-12-31T02:20:30.973453Z",
+     "shell.execute_reply": "2024-12-31T02:20:30.972863Z"
     }
    },
    "outputs": [],
@@ -12224,10 +12224,10 @@
    "execution_count": 32,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:13.543284Z",
-     "iopub.status.busy": "2024-12-14T01:01:13.543099Z",
-     "iopub.status.idle": "2024-12-14T01:01:13.603349Z",
-     "shell.execute_reply": "2024-12-14T01:01:13.602719Z"
+     "iopub.execute_input": "2024-12-31T02:20:30.976050Z",
+     "iopub.status.busy": "2024-12-31T02:20:30.975664Z",
+     "iopub.status.idle": "2024-12-31T02:20:31.035519Z",
+     "shell.execute_reply": "2024-12-31T02:20:31.034922Z"
     }
    },
    "outputs": [
@@ -13313,9 +13313,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"f21b691b-6def-4049-8b21-6c417fc5a93c\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"f21b691b-6def-4049-8b21-6c417fc5a93c\")) {                    Plotly.newPlot(                        \"f21b691b-6def-4049-8b21-6c417fc5a93c\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.0006398981786333025,0.005319283809512854,0.0011588380439206958,-5.923740172875114e-05,-0.0010669109178707004,0.005080018658190966,-0.0030812176410108805,-0.0020524668507277966,-0.0014406939735636115,0.003492556745186448,-0.0025678491219878197,-0.0009152094717137516],[-0.0007601853576488793,0.0001696908293524757,0.00012250228610355407,-0.00034918184974230826,1.5405081285280176e-05,0.005008597858250141,-0.0002970260684378445,-0.0014443857362493873,-0.001099835499189794,0.00047436909517273307,5.108177356305532e-05,-0.003494771895930171],[-0.0007234682561829686,0.0017475537024438381,-0.0001551919267512858,5.749216506956145e-05,-9.699495421955362e-05,-0.00042355581535957754,-0.000791196885984391,0.0002723913639783859,0.00010212998313363642,0.00042187771759927273,0.00015183570212684572,-0.0007432028651237488],[0.11458060890436172,0.00021201284835115075,-0.000942025741096586,0.0004294627869967371,0.02004331909120083,0.002104991814121604,7.662265852559358e-05,-0.0015435622772201896,-0.0008476486545987427,-0.0005817683413624763,0.00011934742360608652,-1.9130493456032127e-05],[-0.001126819639466703,0.0012377764796838164,-0.0012320708483457565,-0.0005953610525466502,-0.0007538085337728262,-0.0005843526450917125,0.004813097883015871,0.00018163899949286133,-0.0005363250384107232,0.0008580865105614066,-0.00029840212664566934,-1.2183103535789996e-05],[-0.004240894690155983,0.0029526741709560156,0.0005220275488682091,0.0009532691328786314,0.0001625084987608716,0.3435087502002716,-0.0003038727736566216,0.00010364028275944293,-0.005300321150571108,0.024866117164492607,0.014383613131940365,-0.0023283488117158413],[-0.0023879888467490673,-0.002172317821532488,-0.00047651707427576184,0.0004340272571425885,-0.004673780873417854,0.001858678413555026,-0.0026543056592345238,0.0014364985981956124,0.000303067296044901,0.1304299533367157,8.957769750850275e-05,0.0011777335312217474],[0.0003194456803612411,0.020570717751979828,0.00031887509976513684,-0.002512404229491949,-0.0002621884341351688,-0.00024587716325186193,0.0005522335413843393,-0.00043134228326380253,0.0002570534124970436,0.008090788498520851,-0.003069269470870495,-0.0004241599526721984],[0.0009768297895789146,0.00039294702582992613,0.0017541655106469989,0.022596266120672226,-4.443644138518721e-05,0.00014159921556711197,0.00958488043397665,-0.0003154182049911469,0.0015266804257407784,0.0011816268088296056,-0.01077402476221323,0.009365785866975784],[0.006314304657280445,-0.001094431965611875,0.011661848053336143,0.0013478941982612014,-0.02918611839413643,0.0038340529426932335,-0.04409320279955864,-0.005031823180615902,0.004822427872568369,0.2766473889350891,-3.1280032999347895e-05,-0.0006619150517508388],[0.09538879990577698,0.025069067254662514,0.014238725416362286,0.01475451048463583,9.914294059854001e-05,-8.977906691143289e-05,0.0508294515311718,-0.5050979852676392,0.00014717054727952927,-0.0016020277980715036,0.06883121281862259,0.0023271406535059214],[0.0013427927624434233,0.009631095454096794,-0.07776330411434174,-0.007728920318186283,-0.0005721024353988469,-0.0029568022582679987,-0.00494667561724782,0.000458796217571944,-0.0006325145368464291,-0.006519974209368229,-0.32048866152763367,-0.002472163178026676]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Patched Head Pattern\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"857c4615-23ce-4e0c-b819-9a1aa3fd5753\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"857c4615-23ce-4e0c-b819-9a1aa3fd5753\")) {                    Plotly.newPlot(                        \"857c4615-23ce-4e0c-b819-9a1aa3fd5753\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.0006398981786333025,0.005319283809512854,0.0011588380439206958,-5.923740172875114e-05,-0.0010669109178707004,0.005080018658190966,-0.0030812176410108805,-0.0020524668507277966,-0.0014406939735636115,0.003492556745186448,-0.0025678491219878197,-0.0009152094717137516],[-0.0007601853576488793,0.0001696908293524757,0.00012250228610355407,-0.00034918184974230826,1.5405081285280176e-05,0.005008597858250141,-0.0002970260684378445,-0.0014443857362493873,-0.001099835499189794,0.00047436909517273307,5.108177356305532e-05,-0.003494771895930171],[-0.0007234682561829686,0.0017475537024438381,-0.0001551919267512858,5.749216506956145e-05,-9.699495421955362e-05,-0.00042355581535957754,-0.000791196885984391,0.0002723913639783859,0.00010212998313363642,0.00042187771759927273,0.00015183570212684572,-0.0007432028651237488],[0.11458060890436172,0.00021201284835115075,-0.000942025741096586,0.0004294627869967371,0.02004331909120083,0.002104991814121604,7.662265852559358e-05,-0.0015435622772201896,-0.0008476486545987427,-0.0005817683413624763,0.00011934742360608652,-1.9130493456032127e-05],[-0.001126819639466703,0.0012377764796838164,-0.0012320708483457565,-0.0005953610525466502,-0.0007538085337728262,-0.0005843526450917125,0.004813097883015871,0.00018163899949286133,-0.0005363250384107232,0.0008580865105614066,-0.00029840212664566934,-1.2183103535789996e-05],[-0.004240894690155983,0.0029526741709560156,0.0005220275488682091,0.0009532691328786314,0.0001625084987608716,0.3435087502002716,-0.0003038727736566216,0.00010364028275944293,-0.005300321150571108,0.024866117164492607,0.014383613131940365,-0.0023283488117158413],[-0.0023879888467490673,-0.002172317821532488,-0.00047651707427576184,0.0004340272571425885,-0.004673780873417854,0.001858678413555026,-0.0026543056592345238,0.0014364985981956124,0.000303067296044901,0.1304299533367157,8.957769750850275e-05,0.0011777335312217474],[0.0003194456803612411,0.020570717751979828,0.00031887509976513684,-0.002512404229491949,-0.0002621884341351688,-0.00024587716325186193,0.0005522335413843393,-0.00043134228326380253,0.0002570534124970436,0.008090788498520851,-0.003069269470870495,-0.0004241599526721984],[0.0009768297895789146,0.00039294702582992613,0.0017541655106469989,0.022596266120672226,-4.443644138518721e-05,0.00014159921556711197,0.00958488043397665,-0.0003154182049911469,0.0015266804257407784,0.0011816268088296056,-0.01077402476221323,0.009365785866975784],[0.006314304657280445,-0.001094431965611875,0.011661848053336143,0.0013478941982612014,-0.02918611839413643,0.0038340529426932335,-0.04409320279955864,-0.005031823180615902,0.004822427872568369,0.2766473889350891,-3.1280032999347895e-05,-0.0006619150517508388],[0.09538879990577698,0.025069067254662514,0.014238725416362286,0.01475451048463583,9.914294059854001e-05,-8.977906691143289e-05,0.0508294515311718,-0.5050979852676392,0.00014717054727952927,-0.0016020277980715036,0.06883121281862259,0.0023271406535059214],[0.0013427927624434233,0.009631095454096794,-0.07776330411434174,-0.007728920318186283,-0.0005721024353988469,-0.0029568022582679987,-0.00494667561724782,0.000458796217571944,-0.0006325145368464291,-0.006519974209368229,-0.32048866152763367,-0.002472163178026676]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Patched Head Pattern\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('f21b691b-6def-4049-8b21-6c417fc5a93c');\n",
+       "var gd = document.getElementById('857c4615-23ce-4e0c-b819-9a1aa3fd5753');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -14650,9 +14650,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"d6f23061-00f8-4a84-9d17-cede03fea051\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"d6f23061-00f8-4a84-9d17-cede03fea051\")) {                    Plotly.newPlot(                        \"d6f23061-00f8-4a84-9d17-cede03fea051\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003eAttention Patch=%{x}\\u003cbr\\u003eOutput Patch=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"L0H0\",\"L0H1\",\"L0H2\",\"L0H3\",\"L0H4\",\"L0H5\",\"L0H6\",\"L0H7\",\"L0H8\",\"L0H9\",\"L0H10\",\"L0H11\",\"L1H0\",\"L1H1\",\"L1H2\",\"L1H3\",\"L1H4\",\"L1H5\",\"L1H6\",\"L1H7\",\"L1H8\",\"L1H9\",\"L1H10\",\"L1H11\",\"L2H0\",\"L2H1\",\"L2H2\",\"L2H3\",\"L2H4\",\"L2H5\",\"L2H6\",\"L2H7\",\"L2H8\",\"L2H9\",\"L2H10\",\"L2H11\",\"L3H0\",\"L3H1\",\"L3H2\",\"L3H3\",\"L3H4\",\"L3H5\",\"L3H6\",\"L3H7\",\"L3H8\",\"L3H9\",\"L3H10\",\"L3H11\",\"L4H0\",\"L4H1\",\"L4H2\",\"L4H3\",\"L4H4\",\"L4H5\",\"L4H6\",\"L4H7\",\"L4H8\",\"L4H9\",\"L4H10\",\"L4H11\",\"L5H0\",\"L5H1\",\"L5H2\",\"L5H3\",\"L5H4\",\"L5H5\",\"L5H6\",\"L5H7\",\"L5H8\",\"L5H9\",\"L5H10\",\"L5H11\",\"L6H0\",\"L6H1\",\"L6H2\",\"L6H3\",\"L6H4\",\"L6H5\",\"L6H6\",\"L6H7\",\"L6H8\",\"L6H9\",\"L6H10\",\"L6H11\",\"L7H0\",\"L7H1\",\"L7H2\",\"L7H3\",\"L7H4\",\"L7H5\",\"L7H6\",\"L7H7\",\"L7H8\",\"L7H9\",\"L7H10\",\"L7H11\",\"L8H0\",\"L8H1\",\"L8H2\",\"L8H3\",\"L8H4\",\"L8H5\",\"L8H6\",\"L8H7\",\"L8H8\",\"L8H9\",\"L8H10\",\"L8H11\",\"L9H0\",\"L9H1\",\"L9H2\",\"L9H3\",\"L9H4\",\"L9H5\",\"L9H6\",\"L9H7\",\"L9H8\",\"L9H9\",\"L9H10\",\"L9H11\",\"L10H0\",\"L10H1\",\"L10H2\",\"L10H3\",\"L10H4\",\"L10H5\",\"L10H6\",\"L10H7\",\"L10H8\",\"L10H9\",\"L10H10\",\"L10H11\",\"L11H0\",\"L11H1\",\"L11H2\",\"L11H3\",\"L11H4\",\"L11H5\",\"L11H6\",\"L11H7\",\"L11H8\",\"L11H9\",\"L11H10\",\"L11H11\"],\"legendgroup\":\"\",\"marker\":{\"color\":\"#636efa\",\"symbol\":\"circle\"},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[0.0006398981786333025,0.005319283809512854,0.0011588380439206958,-5.923740172875114e-05,-0.0010669109178707004,0.005080018658190966,-0.0030812176410108805,-0.0020524668507277966,-0.0014406939735636115,0.003492556745186448,-0.0025678491219878197,-0.0009152094717137516,-0.0007601853576488793,0.0001696908293524757,0.00012250228610355407,-0.00034918184974230826,1.5405081285280176e-05,0.005008597858250141,-0.0002970260684378445,-0.0014443857362493873,-0.001099835499189794,0.00047436909517273307,5.108177356305532e-05,-0.003494771895930171,-0.0007234682561829686,0.0017475537024438381,-0.0001551919267512858,5.749216506956145e-05,-9.699495421955362e-05,-0.00042355581535957754,-0.000791196885984391,0.0002723913639783859,0.00010212998313363642,0.00042187771759927273,0.00015183570212684572,-0.0007432028651237488,0.11458060890436172,0.00021201284835115075,-0.000942025741096586,0.0004294627869967371,0.02004331909120083,0.002104991814121604,7.662265852559358e-05,-0.0015435622772201896,-0.0008476486545987427,-0.0005817683413624763,0.00011934742360608652,-1.9130493456032127e-05,-0.001126819639466703,0.0012377764796838164,-0.0012320708483457565,-0.0005953610525466502,-0.0007538085337728262,-0.0005843526450917125,0.004813097883015871,0.00018163899949286133,-0.0005363250384107232,0.0008580865105614066,-0.00029840212664566934,-1.2183103535789996e-05,-0.004240894690155983,0.0029526741709560156,0.0005220275488682091,0.0009532691328786314,0.0001625084987608716,0.3435087502002716,-0.0003038727736566216,0.00010364028275944293,-0.005300321150571108,0.024866117164492607,0.014383613131940365,-0.0023283488117158413,-0.0023879888467490673,-0.002172317821532488,-0.00047651707427576184,0.0004340272571425885,-0.004673780873417854,0.001858678413555026,-0.0026543056592345238,0.0014364985981956124,0.000303067296044901,0.1304299533367157,8.957769750850275e-05,0.0011777335312217474,0.0003194456803612411,0.020570717751979828,0.00031887509976513684,-0.002512404229491949,-0.0002621884341351688,-0.00024587716325186193,0.0005522335413843393,-0.00043134228326380253,0.0002570534124970436,0.008090788498520851,-0.003069269470870495,-0.0004241599526721984,0.0009768297895789146,0.00039294702582992613,0.0017541655106469989,0.022596266120672226,-4.443644138518721e-05,0.00014159921556711197,0.00958488043397665,-0.0003154182049911469,0.0015266804257407784,0.0011816268088296056,-0.01077402476221323,0.009365785866975784,0.006314304657280445,-0.001094431965611875,0.011661848053336143,0.0013478941982612014,-0.02918611839413643,0.0038340529426932335,-0.04409320279955864,-0.005031823180615902,0.004822427872568369,0.2766473889350891,-3.1280032999347895e-05,-0.0006619150517508388,0.09538879990577698,0.025069067254662514,0.014238725416362286,0.01475451048463583,9.914294059854001e-05,-8.977906691143289e-05,0.0508294515311718,-0.5050979852676392,0.00014717054727952927,-0.0016020277980715036,0.06883121281862259,0.0023271406535059214,0.0013427927624434233,0.009631095454096794,-0.07776330411434174,-0.007728920318186283,-0.0005721024353988469,-0.0029568022582679987,-0.00494667561724782,0.000458796217571944,-0.0006325145368464291,-0.006519974209368229,-0.32048866152763367,-0.002472163178026676],\"xaxis\":\"x\",\"y\":[0.000949342327658087,0.016124119982123375,0.0018556242575868964,0.0034389242064207792,-0.009823138825595379,0.011059673503041267,-0.0040638200007379055,-0.0015785341383889318,-0.001208073808811605,0.003828749991953373,-0.004257441032677889,-0.0011428623693063855,-0.001076644053682685,-0.0003779782564379275,3.0206042538338806e-06,-0.000260510336374864,-0.00014149851631373167,0.0038328110240399837,-0.0004281202855054289,-0.0014291484840214252,-0.0009217541082762182,0.0006938663427717984,0.00043359093251638114,-0.0035708239302039146,-0.0004966544220224023,0.0008070718613453209,0.0005420306115411222,-0.0005315592279657722,-0.0007152119651436806,-0.0010380810126662254,-0.0009492752142250538,-8.538240945199504e-05,0.0002774928288999945,0.0021076432894915342,-0.0001976146304514259,-0.0016398524167016149,0.11626167595386505,0.00025017315056174994,-0.0014669395750388503,-0.0003969745012000203,0.01896214485168457,-0.00018882131553255022,0.011170395649969578,-0.0013304082676768303,-0.0007363561890088022,-0.00030310085276141763,-0.00014579448907170445,-0.0002225178322987631,-0.0016502231592312455,0.00029299859306775033,-0.0014352232683449984,0.030841777101159096,-0.007431894540786743,-0.0002820908557623625,0.006017110776156187,-0.011006947606801987,-0.0012651296565309167,0.001490030437707901,-0.00018016225658357143,0.0029450219590216875,-0.004210856277495623,0.0029608632903546095,0.002045620232820511,0.0013410139363259077,-0.0012189144035801291,0.34349939227104187,0.0005627049831673503,-0.0001257913827430457,-0.005152345169335604,0.016241183504462242,0.017089638859033585,-0.004173669498413801,0.039775215089321136,0.015227536670863628,-0.001023011514917016,0.0008081123232841492,-0.004935599863529205,-0.002123551908880472,-0.014273260720074177,0.0013749118661507964,0.0014838549541309476,0.1302708089351654,-0.00033538774005137384,0.0012914760736748576,0.0003724069392774254,0.0195147804915905,0.00022308839834295213,0.12424899637699127,-0.0004041232750751078,-0.007652163505554199,0.0013012762647122145,-0.0011244030902162194,-0.007449581753462553,0.19224178791046143,-0.0032753082923591137,-0.000501957256346941,-0.001007740618661046,3.1212908652378246e-05,-0.0008586570620536804,0.012360110878944397,-0.00040429108776152134,-0.004328995477408171,0.31855517625808716,0.002329523442313075,0.002118081087246537,0.0001410957775078714,0.27793484926223755,0.005738543812185526,0.0058897752314805984,-0.000968372100032866,0.009125681594014168,0.020675498992204666,-0.03700448200106621,0.014263628050684929,-0.04828385263681412,0.05834108963608742,0.0006516450084745884,0.2636081576347351,0.0004918886115774512,-0.0026110773906111717,0.08374276012182236,0.020676370710134506,-0.0037437702994793653,0.01085111778229475,-0.0010960765648633242,0.0004738992138300091,0.04817944020032883,-0.47990989685058594,0.00018412260396871716,0.011861777864396572,0.06088537722826004,0.0008459034143015742,0.005328110884875059,-0.011493667028844357,-0.11350765824317932,0.006330112461000681,0.0003166264505125582,-0.0011600126745179296,-0.022669030353426933,0.004070901777595282,0.007316238712519407,-0.008345123380422592,-0.27817434072494507,0.003635196480900049],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Attention Patch\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Output Patch\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Scatter plot of output patching vs attention patching\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"a4bdc2e4-754f-45ec-9243-67a82b72f8a2\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"a4bdc2e4-754f-45ec-9243-67a82b72f8a2\")) {                    Plotly.newPlot(                        \"a4bdc2e4-754f-45ec-9243-67a82b72f8a2\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003eAttention Patch=%{x}\\u003cbr\\u003eOutput Patch=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"L0H0\",\"L0H1\",\"L0H2\",\"L0H3\",\"L0H4\",\"L0H5\",\"L0H6\",\"L0H7\",\"L0H8\",\"L0H9\",\"L0H10\",\"L0H11\",\"L1H0\",\"L1H1\",\"L1H2\",\"L1H3\",\"L1H4\",\"L1H5\",\"L1H6\",\"L1H7\",\"L1H8\",\"L1H9\",\"L1H10\",\"L1H11\",\"L2H0\",\"L2H1\",\"L2H2\",\"L2H3\",\"L2H4\",\"L2H5\",\"L2H6\",\"L2H7\",\"L2H8\",\"L2H9\",\"L2H10\",\"L2H11\",\"L3H0\",\"L3H1\",\"L3H2\",\"L3H3\",\"L3H4\",\"L3H5\",\"L3H6\",\"L3H7\",\"L3H8\",\"L3H9\",\"L3H10\",\"L3H11\",\"L4H0\",\"L4H1\",\"L4H2\",\"L4H3\",\"L4H4\",\"L4H5\",\"L4H6\",\"L4H7\",\"L4H8\",\"L4H9\",\"L4H10\",\"L4H11\",\"L5H0\",\"L5H1\",\"L5H2\",\"L5H3\",\"L5H4\",\"L5H5\",\"L5H6\",\"L5H7\",\"L5H8\",\"L5H9\",\"L5H10\",\"L5H11\",\"L6H0\",\"L6H1\",\"L6H2\",\"L6H3\",\"L6H4\",\"L6H5\",\"L6H6\",\"L6H7\",\"L6H8\",\"L6H9\",\"L6H10\",\"L6H11\",\"L7H0\",\"L7H1\",\"L7H2\",\"L7H3\",\"L7H4\",\"L7H5\",\"L7H6\",\"L7H7\",\"L7H8\",\"L7H9\",\"L7H10\",\"L7H11\",\"L8H0\",\"L8H1\",\"L8H2\",\"L8H3\",\"L8H4\",\"L8H5\",\"L8H6\",\"L8H7\",\"L8H8\",\"L8H9\",\"L8H10\",\"L8H11\",\"L9H0\",\"L9H1\",\"L9H2\",\"L9H3\",\"L9H4\",\"L9H5\",\"L9H6\",\"L9H7\",\"L9H8\",\"L9H9\",\"L9H10\",\"L9H11\",\"L10H0\",\"L10H1\",\"L10H2\",\"L10H3\",\"L10H4\",\"L10H5\",\"L10H6\",\"L10H7\",\"L10H8\",\"L10H9\",\"L10H10\",\"L10H11\",\"L11H0\",\"L11H1\",\"L11H2\",\"L11H3\",\"L11H4\",\"L11H5\",\"L11H6\",\"L11H7\",\"L11H8\",\"L11H9\",\"L11H10\",\"L11H11\"],\"legendgroup\":\"\",\"marker\":{\"color\":\"#636efa\",\"symbol\":\"circle\"},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[0.0006398981786333025,0.005319283809512854,0.0011588380439206958,-5.923740172875114e-05,-0.0010669109178707004,0.005080018658190966,-0.0030812176410108805,-0.0020524668507277966,-0.0014406939735636115,0.003492556745186448,-0.0025678491219878197,-0.0009152094717137516,-0.0007601853576488793,0.0001696908293524757,0.00012250228610355407,-0.00034918184974230826,1.5405081285280176e-05,0.005008597858250141,-0.0002970260684378445,-0.0014443857362493873,-0.001099835499189794,0.00047436909517273307,5.108177356305532e-05,-0.003494771895930171,-0.0007234682561829686,0.0017475537024438381,-0.0001551919267512858,5.749216506956145e-05,-9.699495421955362e-05,-0.00042355581535957754,-0.000791196885984391,0.0002723913639783859,0.00010212998313363642,0.00042187771759927273,0.00015183570212684572,-0.0007432028651237488,0.11458060890436172,0.00021201284835115075,-0.000942025741096586,0.0004294627869967371,0.02004331909120083,0.002104991814121604,7.662265852559358e-05,-0.0015435622772201896,-0.0008476486545987427,-0.0005817683413624763,0.00011934742360608652,-1.9130493456032127e-05,-0.001126819639466703,0.0012377764796838164,-0.0012320708483457565,-0.0005953610525466502,-0.0007538085337728262,-0.0005843526450917125,0.004813097883015871,0.00018163899949286133,-0.0005363250384107232,0.0008580865105614066,-0.00029840212664566934,-1.2183103535789996e-05,-0.004240894690155983,0.0029526741709560156,0.0005220275488682091,0.0009532691328786314,0.0001625084987608716,0.3435087502002716,-0.0003038727736566216,0.00010364028275944293,-0.005300321150571108,0.024866117164492607,0.014383613131940365,-0.0023283488117158413,-0.0023879888467490673,-0.002172317821532488,-0.00047651707427576184,0.0004340272571425885,-0.004673780873417854,0.001858678413555026,-0.0026543056592345238,0.0014364985981956124,0.000303067296044901,0.1304299533367157,8.957769750850275e-05,0.0011777335312217474,0.0003194456803612411,0.020570717751979828,0.00031887509976513684,-0.002512404229491949,-0.0002621884341351688,-0.00024587716325186193,0.0005522335413843393,-0.00043134228326380253,0.0002570534124970436,0.008090788498520851,-0.003069269470870495,-0.0004241599526721984,0.0009768297895789146,0.00039294702582992613,0.0017541655106469989,0.022596266120672226,-4.443644138518721e-05,0.00014159921556711197,0.00958488043397665,-0.0003154182049911469,0.0015266804257407784,0.0011816268088296056,-0.01077402476221323,0.009365785866975784,0.006314304657280445,-0.001094431965611875,0.011661848053336143,0.0013478941982612014,-0.02918611839413643,0.0038340529426932335,-0.04409320279955864,-0.005031823180615902,0.004822427872568369,0.2766473889350891,-3.1280032999347895e-05,-0.0006619150517508388,0.09538879990577698,0.025069067254662514,0.014238725416362286,0.01475451048463583,9.914294059854001e-05,-8.977906691143289e-05,0.0508294515311718,-0.5050979852676392,0.00014717054727952927,-0.0016020277980715036,0.06883121281862259,0.0023271406535059214,0.0013427927624434233,0.009631095454096794,-0.07776330411434174,-0.007728920318186283,-0.0005721024353988469,-0.0029568022582679987,-0.00494667561724782,0.000458796217571944,-0.0006325145368464291,-0.006519974209368229,-0.32048866152763367,-0.002472163178026676],\"xaxis\":\"x\",\"y\":[0.000949342327658087,0.016124119982123375,0.0018556242575868964,0.0034389242064207792,-0.009823138825595379,0.011059673503041267,-0.0040638200007379055,-0.0015785341383889318,-0.001208073808811605,0.003828749991953373,-0.004257441032677889,-0.0011428623693063855,-0.001076644053682685,-0.0003779782564379275,3.0206042538338806e-06,-0.000260510336374864,-0.00014149851631373167,0.0038328110240399837,-0.0004281202855054289,-0.0014291484840214252,-0.0009217541082762182,0.0006938663427717984,0.00043359093251638114,-0.0035708239302039146,-0.0004966544220224023,0.0008070718613453209,0.0005420306115411222,-0.0005315592279657722,-0.0007152119651436806,-0.0010380810126662254,-0.0009492752142250538,-8.538240945199504e-05,0.0002774928288999945,0.0021076432894915342,-0.0001976146304514259,-0.0016398524167016149,0.11626167595386505,0.00025017315056174994,-0.0014669395750388503,-0.0003969745012000203,0.01896214485168457,-0.00018882131553255022,0.011170395649969578,-0.0013304082676768303,-0.0007363561890088022,-0.00030310085276141763,-0.00014579448907170445,-0.0002225178322987631,-0.0016502231592312455,0.00029299859306775033,-0.0014352232683449984,0.030841777101159096,-0.007431894540786743,-0.0002820908557623625,0.006017110776156187,-0.011006947606801987,-0.0012651296565309167,0.001490030437707901,-0.00018016225658357143,0.0029450219590216875,-0.004210856277495623,0.0029608632903546095,0.002045620232820511,0.0013410139363259077,-0.0012189144035801291,0.34349939227104187,0.0005627049831673503,-0.0001257913827430457,-0.005152345169335604,0.016241183504462242,0.017089638859033585,-0.004173669498413801,0.039775215089321136,0.015227536670863628,-0.001023011514917016,0.0008081123232841492,-0.004935599863529205,-0.002123551908880472,-0.014273260720074177,0.0013749118661507964,0.0014838549541309476,0.1302708089351654,-0.00033538774005137384,0.0012914760736748576,0.0003724069392774254,0.0195147804915905,0.00022308839834295213,0.12424899637699127,-0.0004041232750751078,-0.007652163505554199,0.0013012762647122145,-0.0011244030902162194,-0.007449581753462553,0.19224178791046143,-0.0032753082923591137,-0.000501957256346941,-0.001007740618661046,3.1212908652378246e-05,-0.0008586570620536804,0.012360110878944397,-0.00040429108776152134,-0.004328995477408171,0.31855517625808716,0.002329523442313075,0.002118081087246537,0.0001410957775078714,0.27793484926223755,0.005738543812185526,0.0058897752314805984,-0.000968372100032866,0.009125681594014168,0.020675498992204666,-0.03700448200106621,0.014263628050684929,-0.04828385263681412,0.05834108963608742,0.0006516450084745884,0.2636081576347351,0.0004918886115774512,-0.0026110773906111717,0.08374276012182236,0.020676370710134506,-0.0037437702994793653,0.01085111778229475,-0.0010960765648633242,0.0004738992138300091,0.04817944020032883,-0.47990989685058594,0.00018412260396871716,0.011861777864396572,0.06088537722826004,0.0008459034143015742,0.005328110884875059,-0.011493667028844357,-0.11350765824317932,0.006330112461000681,0.0003166264505125582,-0.0011600126745179296,-0.022669030353426933,0.004070901777595282,0.007316238712519407,-0.008345123380422592,-0.27817434072494507,0.003635196480900049],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Attention Patch\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Output Patch\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Scatter plot of output patching vs attention patching\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('d6f23061-00f8-4a84-9d17-cede03fea051');\n",
+       "var gd = document.getElementById('a4bdc2e4-754f-45ec-9243-67a82b72f8a2');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -14731,37 +14731,37 @@
    "execution_count": 33,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:13.605613Z",
-     "iopub.status.busy": "2024-12-14T01:01:13.605390Z",
-     "iopub.status.idle": "2024-12-14T01:01:13.618831Z",
-     "shell.execute_reply": "2024-12-14T01:01:13.618283Z"
+     "iopub.execute_input": "2024-12-31T02:20:31.038008Z",
+     "iopub.status.busy": "2024-12-31T02:20:31.037656Z",
+     "iopub.status.idle": "2024-12-31T02:20:31.050109Z",
+     "shell.execute_reply": "2024-12-31T02:20:31.049558Z"
     }
    },
    "outputs": [
     {
      "data": {
       "text/html": [
-       "<div style='max-width: 700px;'><h2>Top Early Heads</h2><br/><div id=\"circuits-vis-cec38c2d-008b\" style=\"margin: 15px 0;\"/>\n",
+       "<div style='max-width: 700px;'><h2>Top Early Heads</h2><br/><div id=\"circuits-vis-42e293d6-6a8c\" style=\"margin: 15px 0;\"/>\n",
        "    <script crossorigin type=\"module\">\n",
        "    import { render, AttentionHeads } from \"https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js\";\n",
        "    render(\n",
-       "      \"circuits-vis-cec38c2d-008b\",\n",
+       "      \"circuits-vis-42e293d6-6a8c\",\n",
        "      AttentionHeads,\n",
        "      {\"attention\": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9964697360992432, 0.0035302219912409782, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9966080188751221, 0.0017803364899009466, 0.0016116512706503272, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9577597975730896, 0.002041348023340106, 0.00576710794121027, 0.03443171828985214, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9969283938407898, 0.0006081241299398243, 0.0016115037724375725, 0.0005201429594308138, 0.0003317868977319449, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9679325819015503, 0.0013422606280073524, 0.011834591627120972, 0.004275884013622999, 0.005249680485576391, 0.009365071542561054, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9869164228439331, 0.0013478520559147, 0.008249778300523758, 0.00044454942690208554, 0.0012886020122095942, 0.0008496184600517154, 0.0009031315566971898, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9841058850288391, 0.005145575385540724, 0.0032505656126886606, 0.00023865938419476151, 0.0014756217133253813, 0.000972211651969701, 0.00134052406065166, 0.003471038304269314, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9905135631561279, 0.0026424615643918514, 0.0020855581387877464, 4.1533587591402465e-07, 1.741811320243869e-05, 0.0003096052969340235, 0.00018154106510337442, 0.0010755121475085616, 0.0031737794633954763, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9528716206550598, 0.00732765905559063, 0.008451228961348534, 0.00028046435909345746, 0.002839941531419754, 0.002773231128230691, 0.001109760720282793, 0.0011476994259282947, 0.0057571399956941605, 0.017441321164369583, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04626865312457085, 0.00023600729764439166, 0.00012578870519064367, 0.5745652914047241, 0.3546755015850067, 0.0192879606038332, 9.036171832121909e-05, 0.00044691169750876725, 0.004133324604481459, 3.63319622920244e-06, 0.00016656972002238035, 0.0, 0.0, 0.0, 0.0], [0.887678861618042, 0.005508160684257746, 0.008590048179030418, 0.022789321839809418, 0.034917183220386505, 0.001486042165197432, 7.143727270886302e-05, 0.0006066859932616353, 0.010601978749036789, 0.0001475940807722509, 0.004989828448742628, 0.022612828761339188, 0.0, 0.0, 0.0], [0.9325922131538391, 0.009143135510385036, 0.0023391752038151026, 0.0008593837847001851, 0.0012484431499615312, 0.00037233170587569475, 0.0001396290463162586, 5.667153527610935e-05, 0.029758313670754433, 0.002666445914655924, 0.00340129598043859, 0.0010437102755531669, 0.016379283741116524, 0.0, 0.0], [0.9817627668380737, 0.001917588640935719, 0.00040775517118163407, 1.3564374512498034e-07, 1.264090315089561e-05, 2.9349219403229654e-05, 3.720399035955779e-05, 1.9605669876909815e-05, 0.00034280188265256584, 0.0020146335009485483, 0.007377031724900007, 4.751105734612793e-06, 0.0004324739275034517, 0.005641316995024681, 0.0], [0.908775269985199, 0.004100896418094635, 0.0067780399695038795, 1.580042953719385e-05, 0.00011193934915354475, 8.643363980809227e-05, 3.187262109349831e-06, 0.001585155026987195, 0.003799399361014366, 0.0067048450000584126, 0.0139771094545722, 0.0001160897736554034, 0.004422870930284262, 0.021179812029004097, 0.028343213722109795]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9990116357803345, 0.0009883929742500186, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9985806941986084, 0.0004404807696118951, 0.0009789254982024431, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9970335960388184, 0.0004637633392121643, 0.0009368518949486315, 0.0015657964395359159, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9892035126686096, 8.868379518389702e-05, 0.000574222591239959, 0.009837915189564228, 0.0002956092939712107, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9954046010971069, 0.00010050175478681922, 0.0013415871653705835, 3.886737977154553e-05, 0.0001206161396112293, 0.00299392850138247, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9989855885505676, 0.00010351995297241956, 0.0004172757326159626, 7.227508376672631e-06, 6.046874841558747e-05, 0.0002549678029026836, 0.0001708982017589733, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9980852603912354, 0.00016822919133119285, 0.00010910424316534773, 4.1012435758602805e-06, 5.103566945763305e-05, 0.0001056204637279734, 4.150096719968133e-05, 0.0014351928839460015, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9972643852233887, 7.076081237755716e-05, 6.327207665890455e-05, 2.615450966914068e-08, 5.106949174660258e-06, 3.7383753806352615e-05, 3.455430851317942e-05, 0.000461774121504277, 0.0020626918412745, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9859043955802917, 0.0015692580491304398, 0.001985961804166436, 5.8834184528677724e-06, 0.0003162050561513752, 0.00014976764214225113, 2.4109986043185927e-05, 0.00012667303963098675, 0.0012655562022700906, 0.00865224003791809, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3212882876396179, 0.0026113800704479218, 0.0017525857547298074, 0.48284193873405457, 0.1752953678369522, 0.012053580023348331, 5.7314944569952786e-05, 0.00047417485620826483, 0.0011855909833684564, 4.6230838051997125e-05, 0.0023935972712934017, 0.0, 0.0, 0.0, 0.0], [0.9928818941116333, 0.0011452743783593178, 0.0015605579828843474, 5.570496796281077e-05, 0.0020913302432745695, 0.0002502372954040766, 9.751672678248724e-07, 1.4541505152010359e-05, 0.0006855707615613937, 3.3003976568579674e-05, 0.0003418456471990794, 0.0009391428320668638, 0.0, 0.0, 0.0], [0.9924144148826599, 0.0006524075288325548, 0.00016266616876237094, 7.948347956698854e-06, 0.00015110620006453246, 6.227314588613808e-05, 7.466151714652369e-07, 4.000507090040628e-07, 0.00244068237952888, 3.030487641808577e-05, 4.814808562514372e-05, 9.094910637941211e-05, 0.003938092850148678, 0.0, 0.0], [0.9971888661384583, 0.00062146270647645, 3.946273500332609e-05, 1.9362538239420246e-08, 1.800630002435355e-06, 1.9080694983131252e-05, 9.158955435850658e-06, 7.050003887343337e-07, 6.766093429178e-05, 0.0008072670316323638, 0.00013612494512926787, 2.204007842010469e-06, 2.116413634212222e-05, 0.0010850975522771478, 0.0], [0.9784426689147949, 0.0038412772119045258, 0.001794080133549869, 9.544757631374523e-06, 0.0001613280619494617, 0.00021571738761849701, 3.438441638081713e-07, 0.0001911988656502217, 0.0018583982018753886, 0.002262116875499487, 0.0028807998169213533, 0.00015137717127799988, 3.800494960159995e-05, 0.004419004078954458, 0.0037341148126870394]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9793283343315125, 0.020671749487519264, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.906032145023346, 0.0005559856654144824, 0.09341184794902802, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9916713237762451, 0.0009248568094335496, 0.0019154187757521868, 0.005488428752869368, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9500631093978882, 0.0015093885594978929, 0.00022879372409079224, 0.0011742664501070976, 0.04702451452612877, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9742831587791443, 0.001334858825430274, 0.00016252745990641415, 0.001139267929829657, 1.2687537491729017e-06, 0.023079030215740204, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9734269380569458, 0.0036638062447309494, 0.0007961394148878753, 0.0020336511079221964, 1.7259832020499744e-05, 0.013762583956122398, 0.006299513392150402, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9740443229675293, 0.0018358685774728656, 0.00031735500670038164, 0.0013147101271897554, 4.377634832053445e-05, 0.00045178644359111786, 0.00035931551246903837, 0.021632865071296692, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8537226319313049, 0.0009385455632582307, 1.8928734789369628e-05, 0.0007116434862837195, 4.0574541344540194e-05, 2.706850500544533e-05, 0.0005688024684786797, 0.01303892396390438, 0.1309327930212021, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9718613028526306, 0.0030212444253265858, 0.0018890955252572894, 0.0058778319507837296, 0.0006584360380657017, 0.002494583837687969, 0.001156346406787634, 0.008928273804485798, 0.0013289419002830982, 0.0027839455287903547, 0.0, 0.0, 0.0, 0.0, 0.0], [0.21994593739509583, 0.0028093564324080944, 0.6635686159133911, 0.007326209452003241, 0.001153560820966959, 0.0018521154997870326, 0.0013647314626723528, 0.008370185270905495, 0.0001247532054549083, 0.002670685760676861, 0.09081388264894485, 0.0, 0.0, 0.0, 0.0], [0.9398402571678162, 0.0037398827262222767, 0.0006579139735549688, 0.0027551481034606695, 1.3964868230686989e-05, 0.002081213751807809, 0.007874594070017338, 0.008091994561254978, 0.002785064745694399, 0.003697305917739868, 0.00012857612455263734, 0.028334153816103935, 0.0, 0.0, 0.0], [0.9256278872489929, 0.003680802881717682, 0.0006159098702482879, 0.004310118965804577, 7.914558227639645e-05, 0.0015225791139528155, 0.0014093437930569053, 0.041066594421863556, 0.0011927677551284432, 0.002704742830246687, 8.953329233918339e-05, 0.00031252068583853543, 0.017388159409165382, 0.0, 0.0], [0.7785975933074951, 0.005006679333746433, 8.588898344896734e-05, 0.0016439110040664673, 3.517184086376801e-05, 0.0007985705742612481, 0.004088219255208969, 0.03206512704491615, 0.02768608182668686, 0.003920219838619232, 2.586934351711534e-05, 0.0010157276410609484, 0.009296733886003494, 0.13573415577411652, 0.0], [0.954503059387207, 0.006336927879601717, 0.00032585320877842605, 0.0035448195412755013, 2.0907922589685768e-05, 0.007449527736753225, 0.009912977926433086, 0.005456207785755396, 0.0005529882037080824, 0.0011178080458194017, 6.70458102831617e-05, 0.003978283144533634, 0.0024476239923387766, 6.584440416190773e-05, 0.00422010850161314]]], \"attentionHeadNames\": [\"L5H5\", \"L6H9\", \"L3H0\"], \"tokens\": [\"<|endoftext|>\", \"When\", \" John\", \" and\", \" Mary\", \" went\", \" to\", \" the\", \" shops\", \",\", \" John\", \" gave\", \" the\", \" bag\", \" to\"]}\n",
        "    )\n",
-       "    </script></div><div style='max-width: 700px;'><h2>Top Middle Heads</h2><br/><div id=\"circuits-vis-4796b044-6a22\" style=\"margin: 15px 0;\"/>\n",
+       "    </script></div><div style='max-width: 700px;'><h2>Top Middle Heads</h2><br/><div id=\"circuits-vis-17a2a2ac-56a8\" style=\"margin: 15px 0;\"/>\n",
        "    <script crossorigin type=\"module\">\n",
        "    import { render, AttentionHeads } from \"https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js\";\n",
        "    render(\n",
-       "      \"circuits-vis-4796b044-6a22\",\n",
+       "      \"circuits-vis-17a2a2ac-56a8\",\n",
        "      AttentionHeads,\n",
        "      {\"attention\": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9883801937103271, 0.011619755066931248, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9756757020950317, 0.006998097989708185, 0.017326191067695618, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.06910058110952377, 0.005814341828227043, 0.9046230316162109, 0.020462043583393097, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8063386082649231, 0.003319838549941778, 0.0537770576775074, 0.11000825464725494, 0.02655618265271187, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9564918875694275, 0.004251283593475819, 0.011477907188236713, 0.011219605803489685, 0.005006618797779083, 0.011552776210010052, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9077332615852356, 0.0036374430637806654, 0.040214236825704575, 0.012946903705596924, 0.019611867144703865, 0.011057589203119278, 0.004798637703061104, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9552405476570129, 0.009812522679567337, 0.01168749388307333, 0.0090243611484766, 0.0034366166219115257, 0.004399922210723162, 0.0033402289263904095, 0.003058257745578885, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9294586777687073, 0.009998402558267117, 0.004449557047337294, 0.006221562158316374, 0.0014876301866024733, 0.018290331587195396, 0.00712206494063139, 0.0036232024431228638, 0.0193486325442791, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6007111668586731, 0.014475381001830101, 0.09337030351161957, 0.01660209521651268, 0.05098922923207283, 0.10098620504140854, 0.051671840250492096, 0.02743108756840229, 0.02603602036833763, 0.017726697027683258, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6264241933822632, 0.012645086273550987, 0.016070131212472916, 0.024194441735744476, 0.029195599257946014, 0.11857090145349503, 0.06370382010936737, 0.023378603160381317, 0.001658876077271998, 0.06528978794813156, 0.018868643790483475, 0.0, 0.0, 0.0, 0.0], [0.2991042137145996, 0.0021427334286272526, 0.024437544867396355, 0.005155442748218775, 0.010526876896619797, 0.003321276046335697, 0.009254480712115765, 0.005843525752425194, 0.0021181809715926647, 0.017681695520877838, 0.6048213839530945, 0.015592672862112522, 0.0, 0.0, 0.0], [0.8641641139984131, 0.004937537480145693, 0.019023817032575607, 0.001670254161581397, 0.00609729764983058, 0.0005472623743116856, 0.003564003622159362, 0.00030002548010088503, 0.007773987948894501, 0.007786957547068596, 0.06421297788619995, 0.005789309274405241, 0.014132493175566196, 0.0, 0.0], [0.9178580045700073, 0.005138528533279896, 0.003403181442990899, 0.0021888096816837788, 0.002614964498206973, 0.0028868599329143763, 0.0028416835702955723, 0.0013745100004598498, 0.006431094836443663, 0.012306848540902138, 0.0090817641466856, 0.01310056634247303, 0.008110282011330128, 0.012662900611758232, 0.0], [0.019487837329506874, 5.700236215488985e-05, 0.026276404038071632, 0.0009103945340029895, 0.0056328438222408295, 7.981786620803177e-05, 5.7736750022741035e-05, 0.000307683803839609, 0.00014904775889590383, 0.0002852973702829331, 0.9430450201034546, 0.0007767666247673333, 0.0018659740453585982, 0.0006072799442335963, 0.0004609254829119891]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9562382698059082, 0.043761663138866425, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9643059372901917, 0.00742809334769845, 0.028266001492738724, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6456211805343628, 0.029121818020939827, 0.20926059782505035, 0.11599643528461456, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7634043097496033, 0.01714019477367401, 0.026512376964092255, 0.08158667385578156, 0.11135648936033249, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8267454504966736, 0.029910113662481308, 0.015236063860356808, 0.034572526812553406, 0.027030833065509796, 0.06650502979755402, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7882238626480103, 0.02833343856036663, 0.04124196991324425, 0.024983419105410576, 0.05788382515311241, 0.020290205255150795, 0.03904324769973755, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7217290997505188, 0.027771806344389915, 0.02081979066133499, 0.024504197761416435, 0.01813969761133194, 0.014079142361879349, 0.04442400485277176, 0.12853237986564636, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7728042602539062, 0.04133640229701996, 0.006504536606371403, 0.007949033752083778, 0.006013217847794294, 0.01562037505209446, 0.03980163484811783, 0.05545476824045181, 0.054515715688467026, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.47073426842689514, 0.03082239255309105, 0.07406120002269745, 0.055252522230148315, 0.14865727722644806, 0.062336407601833344, 0.034053124487400055, 0.026126855984330177, 0.042438995093107224, 0.05551688000559807, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6962025165557861, 0.0052972170524299145, 0.004229621961712837, 0.05190793052315712, 0.007751153316348791, 0.03517890349030495, 0.03057355433702469, 0.021498825401067734, 0.01566012017428875, 0.051384858787059784, 0.08031534403562546, 0.0, 0.0, 0.0, 0.0], [0.37817656993865967, 0.006224599666893482, 0.023588571697473526, 0.044115372002124786, 0.059451501816511154, 0.00953902117908001, 0.012997719459235668, 0.014349035918712616, 0.0203937329351902, 0.01785479485988617, 0.33459627628326416, 0.07871285080909729, 0.0, 0.0, 0.0], [0.4760020971298218, 0.011026055552065372, 0.018324486911296844, 0.04565680772066116, 0.026791490614414215, 0.011260150000452995, 0.018724240362644196, 0.018448904156684875, 0.04068983346223831, 0.023096060380339622, 0.12447667866945267, 0.08994489908218384, 0.0955582857131958, 0.0, 0.0], [0.7641531229019165, 0.00824385043233633, 0.0032178766559809446, 0.004948926158249378, 0.004036621656268835, 0.00750678451731801, 0.011691190302371979, 0.013843986205756664, 0.01143654901534319, 0.0222361721098423, 0.013085336424410343, 0.03330926224589348, 0.036730363965034485, 0.06555990874767303, 0.0], [0.1303366869688034, 0.0022374859545379877, 0.0373632088303566, 0.06385007500648499, 0.11956273019313812, 0.011286774650216103, 0.006565204821527004, 0.00851503573358059, 0.005633368622511625, 0.007633764296770096, 0.5373668074607849, 0.0301155224442482, 0.020915960893034935, 0.005129658151417971, 0.013487689197063446]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.972596287727356, 0.027403652667999268, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9560405015945435, 0.037302762269973755, 0.006656836252659559, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5009381771087646, 0.0898313894867897, 0.34842997789382935, 0.06080043688416481, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7674180269241333, 0.17525732517242432, 0.021164551377296448, 0.02390032634139061, 0.012259737588465214, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7222559452056885, 0.18388347327709198, 0.023735947906970978, 0.007100591901689768, 0.007225400768220425, 0.05579861253499985, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6940793395042419, 0.13069462776184082, 0.028726812452077866, 0.009216434322297573, 0.011527394875884056, 0.08933128416538239, 0.036424122750759125, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6924421191215515, 0.12459339946508408, 0.01531891617923975, 0.007925059646368027, 0.006713555194437504, 0.0796525627374649, 0.04910353198647499, 0.02425084263086319, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6969695091247559, 0.10134164988994598, 0.011634294874966145, 0.007826745510101318, 0.0032780379988253117, 0.042647555470466614, 0.05842007324099541, 0.042913682758808136, 0.03496843948960304, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6869087815284729, 0.023936893790960312, 0.10650163143873215, 0.02047020196914673, 0.02552187815308571, 0.04722552374005318, 0.022190820425748825, 0.007246002089232206, 0.019754212349653244, 0.04024401307106018, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5810369849205017, 0.011352411471307278, 0.0033976146951317787, 0.0365644209086895, 0.020549381151795387, 0.051692016422748566, 0.024651870131492615, 0.015468521043658257, 0.049509111791849136, 0.15767459571361542, 0.04810298979282379, 0.0, 0.0, 0.0, 0.0], [0.43488532304763794, 0.006897220853716135, 0.012041947804391384, 0.003814741736277938, 0.010545267723500729, 0.0076634688302874565, 0.007842864841222763, 0.006417531985789537, 0.044214341789484024, 0.10391034185886383, 0.3332052230834961, 0.028561806306242943, 0.0, 0.0, 0.0], [0.4747830927371979, 0.024993136525154114, 0.009554888121783733, 0.0037503365892916918, 0.008160687983036041, 0.02339756116271019, 0.010678932070732117, 0.0014935131184756756, 0.030863573774695396, 0.19498929381370544, 0.15084564685821533, 0.054150503128767014, 0.012338940985500813, 0.0, 0.0], [0.7144870758056641, 0.009456266649067402, 0.0035061524249613285, 0.0029858576599508524, 0.001229096669703722, 0.01199608389288187, 0.008696629665791988, 0.007586767431348562, 0.009631272405385971, 0.06210443004965782, 0.014414231292903423, 0.07928328961133957, 0.04383841156959534, 0.030784491449594498, 0.0], [0.1013382077217102, 0.0038837045431137085, 0.01792380027472973, 0.0038827748503535986, 0.009177407249808311, 0.01316938828676939, 0.0034797503612935543, 0.0019711176864802837, 0.004929723683744669, 0.1296602189540863, 0.6630275249481201, 0.0330243818461895, 0.0089839156717062, 0.0017172213410958648, 0.003830986563116312]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.97651207447052, 0.023487908765673637, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9743125438690186, 0.009410635568201542, 0.016276877373456955, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5441411137580872, 0.019679147750139236, 0.33595189452171326, 0.10022785514593124, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8594159483909607, 0.021821973845362663, 0.025710705667734146, 0.03842515870928764, 0.05462610721588135, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7619735598564148, 0.07210072875022888, 0.037441689521074295, 0.04506158083677292, 0.031744565814733505, 0.05167779326438904, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8025401830673218, 0.026160430163145065, 0.033131979405879974, 0.02643769420683384, 0.03139319270849228, 0.04932845011353493, 0.031008044257760048, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7639790773391724, 0.021559985354542732, 0.016495496034622192, 0.025408243760466576, 0.027263421565294266, 0.06810054928064346, 0.047422830015420914, 0.029770376160740852, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7543362379074097, 0.01597309671342373, 0.008247380144894123, 0.00967292208224535, 0.013097813352942467, 0.03411843255162239, 0.03947027027606964, 0.032298993319272995, 0.09278487414121628, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5856387615203857, 0.0213167741894722, 0.026237597689032555, 0.020694918930530548, 0.02824069745838642, 0.11104536801576614, 0.047170888632535934, 0.022986045107245445, 0.04925474151968956, 0.08741410821676254, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6466761231422424, 0.015195527113974094, 0.00594041682779789, 0.02527434751391411, 0.010863534174859524, 0.08686516433954239, 0.032730914652347565, 0.021324681118130684, 0.04200880602002144, 0.09539096057415009, 0.017729468643665314, 0.0, 0.0, 0.0, 0.0], [0.42771559953689575, 0.014421668834984303, 0.024129344150424004, 0.026094987988471985, 0.01488838903605938, 0.010527816601097584, 0.012212568894028664, 0.015342887490987778, 0.017334001138806343, 0.04439026489853859, 0.28982794284820557, 0.10311443358659744, 0.0, 0.0, 0.0], [0.4405166506767273, 0.018903780728578568, 0.010929570533335209, 0.015466002747416496, 0.013489533215761185, 0.020115477964282036, 0.019655633717775345, 0.005925851874053478, 0.022772811353206635, 0.06206439808011055, 0.16126078367233276, 0.17319339513778687, 0.03570614382624626, 0.0, 0.0], [0.7304301261901855, 0.008421944454312325, 0.0022352896630764008, 0.004027598071843386, 0.0036414540372788906, 0.009633221663534641, 0.0213489830493927, 0.021966980770230293, 0.02610298991203308, 0.05093422904610634, 0.0083627263084054, 0.023242846131324768, 0.038515783846378326, 0.051135752350091934, 0.0], [0.16577893495559692, 0.004480816423892975, 0.0077864062041044235, 0.007284588646143675, 0.01337976660579443, 0.019251490011811256, 0.010158197022974491, 0.01172470673918724, 0.011870641261339188, 0.034697193652391434, 0.1448296457529068, 0.4143996834754944, 0.08938297629356384, 0.019558662548661232, 0.04541636258363724]]], \"attentionHeadNames\": [\"L8H6\", \"L8H10\", \"L7H9\", \"L7H3\"], \"tokens\": [\"<|endoftext|>\", \"When\", \" John\", \" and\", \" Mary\", \" went\", \" to\", \" the\", \" shops\", \",\", \" John\", \" gave\", \" the\", \" bag\", \" to\"]}\n",
        "    )\n",
-       "    </script></div><div style='max-width: 700px;'><h2>Top Late Heads</h2><br/><div id=\"circuits-vis-3f1d1fde-8de5\" style=\"margin: 15px 0;\"/>\n",
+       "    </script></div><div style='max-width: 700px;'><h2>Top Late Heads</h2><br/><div id=\"circuits-vis-abec0760-a70a\" style=\"margin: 15px 0;\"/>\n",
        "    <script crossorigin type=\"module\">\n",
        "    import { render, AttentionHeads } from \"https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js\";\n",
        "    render(\n",
-       "      \"circuits-vis-3f1d1fde-8de5\",\n",
+       "      \"circuits-vis-abec0760-a70a\",\n",
        "      AttentionHeads,\n",
        "      {\"attention\": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9628916382789612, 0.03710832819342613, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9778335094451904, 0.0034864130429923534, 0.018680110573768616, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8737697601318359, 0.005964328069239855, 0.08126737922430038, 0.03899851813912392, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8745138645172119, 0.017276111990213394, 0.018874678760766983, 0.054268334060907364, 0.035066962242126465, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9704693555831909, 0.0035177830141037703, 0.0003239834331907332, 0.004373463336378336, 0.0003953034174628556, 0.02092011272907257, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7694658041000366, 0.006685878615826368, 0.011966485530138016, 0.040966276079416275, 0.03344443440437317, 0.04401002824306488, 0.0934610366821289, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8177722692489624, 0.009112595580518246, 0.0042336746118962765, 0.03900325670838356, 0.012653318233788013, 0.007337852846831083, 0.07410021126270294, 0.0357869453728199, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8309933543205261, 0.018796611577272415, 0.00330327614210546, 0.01983213610947132, 0.004009497817605734, 0.012078307569026947, 0.03246007487177849, 0.010322270914912224, 0.06820447742938995, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.31403300166130066, 0.01522789802402258, 0.19789263606071472, 0.059379592537879944, 0.3236042559146881, 0.009037697687745094, 0.019917670637369156, 0.012475269846618176, 0.013939663767814636, 0.03449229896068573, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7319568991661072, 0.03087025322020054, 0.005064091179519892, 0.08703826367855072, 0.004982746671885252, 0.044725559651851654, 0.02350614219903946, 0.0063977413810789585, 0.0034696452785283327, 0.048653144389390945, 0.013335512951016426, 0.0, 0.0, 0.0, 0.0], [0.13710583746433258, 0.012017901986837387, 0.14592327177524567, 0.015683427453041077, 0.44125232100486755, 0.004342416767030954, 0.011117888614535332, 0.007466601673513651, 0.0012341630645096302, 0.01946023479104042, 0.16842849552631378, 0.035967420786619186, 0.0, 0.0, 0.0], [0.4052484929561615, 0.009720955975353718, 0.03158125653862953, 0.09247038513422012, 0.048808369785547256, 0.008174203336238861, 0.031708307564258575, 0.02787904627621174, 0.14128026366233826, 0.0302340779453516, 0.03392859548330307, 0.03171432018280029, 0.10725171118974686, 0.0, 0.0], [0.808469295501709, 0.0056702690199017525, 0.006755992770195007, 0.008696584962308407, 0.010473440401256084, 0.003464736510068178, 0.006760997697710991, 0.0025889205280691385, 0.015097705647349358, 0.01603602059185505, 0.011288060806691647, 0.017277495935559273, 0.00779919745400548, 0.07962135970592499, 0.0], [0.02939111366868019, 0.0016060526249930263, 0.06005561351776123, 0.008175727911293507, 0.8099164366722107, 0.00039796155760996044, 0.0028282543644309044, 0.0033632616978138685, 0.0001569116284372285, 0.0036459483671933413, 0.05494074895977974, 0.001666039228439331, 0.015368940308690071, 5.016689101466909e-05, 0.008436810225248337]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9124024510383606, 0.08759760111570358, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.97407466173172, 0.01434413529932499, 0.011581187136471272, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8240247964859009, 0.03700513392686844, 0.06643965095281601, 0.07253041118383408, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5370795726776123, 0.15483324229717255, 0.0903128981590271, 0.12131187319755554, 0.0964624434709549, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9388002157211304, 0.028733234852552414, 0.0003431019140407443, 0.010481251403689384, 0.000839930260553956, 0.02080223336815834, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6318807005882263, 0.059302832931280136, 0.03517911583185196, 0.0317409411072731, 0.1174689382314682, 0.06869616359472275, 0.05573126673698425, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7625216841697693, 0.04435169696807861, 0.010552966967225075, 0.034276288002729416, 0.014203979633748531, 0.012703591957688332, 0.07259687036275864, 0.048792947083711624, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5725991725921631, 0.12681783735752106, 0.03727060556411743, 0.0431414395570755, 0.04129580408334732, 0.046447426080703735, 0.029164383187890053, 0.03751189261674881, 0.06575141847133636, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12316936254501343, 0.026347072795033455, 0.27960509061813354, 0.041989635676145554, 0.44105827808380127, 0.006138973869383335, 0.007793003693223, 0.007175281643867493, 0.011286810971796513, 0.055436521768569946, 0.0, 0.0, 0.0, 0.0, 0.0], [0.44855940341949463, 0.1432485729455948, 0.010774216614663601, 0.07855041325092316, 0.01404787041246891, 0.014864671975374222, 0.008633698336780071, 0.0030146560166031122, 0.012792127206921577, 0.16408587992191315, 0.10142850130796432, 0.0, 0.0, 0.0, 0.0], [0.10228201746940613, 0.0238832775503397, 0.055250268429517746, 0.0520680733025074, 0.5759841799736023, 0.0011948152678087354, 0.0013662497512996197, 0.0025677781086415052, 0.0018699930515140295, 0.043160032480955124, 0.10887196660041809, 0.03150142729282379, 0.0, 0.0, 0.0], [0.39313194155693054, 0.04406267777085304, 0.04009911045432091, 0.07407406717538834, 0.04061643034219742, 0.007346798665821552, 0.009507199749350548, 0.013865168206393719, 0.0640539675951004, 0.04894229769706726, 0.0627753809094429, 0.0999651551246643, 0.10155977308750153, 0.0, 0.0], [0.7573198676109314, 0.015095721930265427, 0.007350177504122257, 0.013156400062143803, 0.005726401228457689, 0.008556416258215904, 0.008282607421278954, 0.010307075455784798, 0.002904881490394473, 0.017329057678580284, 0.015188980847597122, 0.0796574279665947, 0.030244464054703712, 0.02888045459985733, 0.0], [0.08579524606466293, 0.023773644119501114, 0.028759371489286423, 0.050303805619478226, 0.674644947052002, 0.0010763936443254352, 0.0011514866491779685, 0.002721975091844797, 0.0005617713322862983, 0.039091579616069794, 0.04861969128251076, 0.015520088374614716, 0.0040845321491360664, 0.00043291584006510675, 0.023462524637579918]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9979304075241089, 0.002069620881229639, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9971864819526672, 0.0010516609763726592, 0.0017618348356336355, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9594857096672058, 0.001310725579969585, 0.03694308176636696, 0.002260456094518304, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9891613125801086, 0.001052629784680903, 0.0048541901633143425, 0.001104357186704874, 0.003827564185485244, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9963598847389221, 0.0007981774397194386, 0.0007764195324853063, 0.00019262291607446969, 0.0002416159404674545, 0.0016313291853293777, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.981286346912384, 0.001863775309175253, 0.006344134453684092, 0.0002960922720376402, 0.004604941233992577, 0.0013966941041871905, 0.00420788861811161, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9950011372566223, 0.0013230532640591264, 0.0007920759962871671, 0.00024644460063427687, 0.0003405151073820889, 0.00016826140927150846, 0.0002837753272615373, 0.0018447580514475703, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9920291900634766, 0.0013539871433749795, 0.0007168339216150343, 9.214139572577551e-05, 0.00013419234892353415, 0.00019706363673321903, 0.00035278062568977475, 0.0002426079590804875, 0.004881155211478472, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5483196377754211, 0.0075838202610611916, 0.28265973925590515, 0.0029685653280466795, 0.14815661311149597, 0.0008172960951924324, 0.0006479909643530846, 0.0017078499076887965, 0.004129356704652309, 0.003009046893566847, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8642559051513672, 0.0036493062507361174, 0.004925842396914959, 0.06957440823316574, 0.038683053106069565, 0.009609837085008621, 0.0008548588375560939, 0.0005643004551529884, 0.003707138355821371, 0.0013952680164948106, 0.002780066104605794, 0.0, 0.0, 0.0, 0.0], [0.30252957344055176, 0.006584456656128168, 0.1401754468679428, 0.030065739527344704, 0.4806697964668274, 0.0005923578282818198, 0.0005309724947437644, 0.0024134721606969833, 0.008505754172801971, 0.0013766245683655143, 0.023764921352267265, 0.0027908117044717073, 0.0, 0.0, 0.0], [0.7773900032043457, 0.003722917754203081, 0.022378109395503998, 0.015139483846724033, 0.017408354207873344, 0.002416662173345685, 0.000709561922121793, 0.0007399603491649032, 0.13850137591362, 0.0023606533650308847, 0.005776867736130953, 0.0028014343697577715, 0.010654616169631481, 0.0, 0.0], [0.9735962152481079, 0.0012812522472813725, 0.002617918187752366, 9.891873924061656e-05, 0.0005098494002595544, 0.00012000725837424397, 0.00045229491661302745, 0.00010779645526781678, 0.002954537281766534, 0.001644388772547245, 0.0012141242623329163, 0.00018637391622178257, 0.0008510759216733277, 0.01436527632176876, 0.0], [0.0990300253033638, 0.000971083587501198, 0.06413238495588303, 0.005246965680271387, 0.8110400438308716, 8.250321843661368e-05, 6.596777529921383e-05, 0.0011991349747404456, 0.0008852760074660182, 0.00024891181965358555, 0.010336406528949738, 7.952339365147054e-05, 0.0049433172680437565, 0.00028918671887367964, 0.0014492359478026628]]], \"attentionHeadNames\": [\"L10H7\", \"L11H10\", \"L9H9\"], \"tokens\": [\"<|endoftext|>\", \"When\", \" John\", \" and\", \" Mary\", \" went\", \" to\", \" the\", \" shops\", \",\", \" John\", \" gave\", \" the\", \" bag\", \" to\"]}\n",
        "    )\n",
@@ -14876,10 +14876,10 @@
    "execution_count": 34,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:13.621272Z",
-     "iopub.status.busy": "2024-12-14T01:01:13.620857Z",
-     "iopub.status.idle": "2024-12-14T01:01:13.781947Z",
-     "shell.execute_reply": "2024-12-14T01:01:13.781299Z"
+     "iopub.execute_input": "2024-12-31T02:20:31.052234Z",
+     "iopub.status.busy": "2024-12-31T02:20:31.051928Z",
+     "iopub.status.idle": "2024-12-31T02:20:31.201540Z",
+     "shell.execute_reply": "2024-12-31T02:20:31.200911Z"
     }
    },
    "outputs": [],
@@ -14898,21 +14898,21 @@
    "execution_count": 35,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:13.784795Z",
-     "iopub.status.busy": "2024-12-14T01:01:13.784431Z",
-     "iopub.status.idle": "2024-12-14T01:01:13.794420Z",
-     "shell.execute_reply": "2024-12-14T01:01:13.793896Z"
+     "iopub.execute_input": "2024-12-31T02:20:31.204135Z",
+     "iopub.status.busy": "2024-12-31T02:20:31.203794Z",
+     "iopub.status.idle": "2024-12-31T02:20:31.214082Z",
+     "shell.execute_reply": "2024-12-31T02:20:31.213507Z"
     }
    },
    "outputs": [
     {
      "data": {
       "text/html": [
-       "<div style='max-width: 800px;'><h2>Induction Heads</h2><br/><div id=\"circuits-vis-2ebcde31-65b5\" style=\"margin: 15px 0;\"/>\n",
+       "<div style='max-width: 800px;'><h2>Induction Heads</h2><br/><div id=\"circuits-vis-eeb72893-5fb3\" style=\"margin: 15px 0;\"/>\n",
        "    <script crossorigin type=\"module\">\n",
        "    import { render, AttentionHeads } from \"https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js\";\n",
        "    render(\n",
-       "      \"circuits-vis-2ebcde31-65b5\",\n",
+       "      \"circuits-vis-eeb72893-5fb3\",\n",
        "      AttentionHeads,\n",
        "      {\"attention\": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9995203018188477, 0.0004796621506102383, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9989193677902222, 0.00011719338363036513, 0.0009634991874918342, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.999607503414154, 4.1751700337044895e-05, 9.65828076004982e-06, 0.00034111665445379913, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9989573955535889, 3.091409962507896e-05, 1.212368533742847e-05, 0.0002459374663885683, 0.0007535542827099562, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9994303584098816, 3.2271585951093584e-05, 2.162250348192174e-05, 0.000438225019024685, 4.1958148358389735e-05, 3.5627250326797366e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.998076319694519, 1.6859989045769908e-05, 2.043985659838654e-06, 6.719817611156031e-05, 1.8109081167949626e-07, 0.00016430420510005206, 0.0016730953939259052, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9989305138587952, 0.0001340252347290516, 1.5985064237611368e-05, 0.000491619051899761, 6.565183241491468e-08, 6.484094228653703e-06, 1.1354626622051e-05, 0.00040989238186739385, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9949487447738647, 0.0005971402861177921, 1.4574082342733163e-05, 0.0018354525091126561, 9.743422424435266e-07, 7.463527435902506e-05, 5.5863079069240484e-06, 0.00020289771782699972, 0.002319993684068322, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9855782985687256, 6.499102164525539e-05, 8.181475277524441e-05, 0.0024699484929442406, 4.049685031759509e-08, 3.498060686979443e-05, 0.001560866367071867, 0.0018861292628571391, 0.00011740217451006174, 0.008205565623939037, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9950633645057678, 0.00021360788377933204, 0.00014613209350500256, 0.00030726491240784526, 2.292262615810614e-05, 0.0002385093830525875, 1.9861789041897282e-05, 0.000960799225140363, 0.0001365413481835276, 0.0016751259099692106, 0.001215968164615333, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9907191395759583, 0.00013381309690885246, 3.014638878084952e-06, 0.0023974282667040825, 2.1563739210250787e-05, 0.0004241704009473324, 1.383451308356598e-05, 5.452497134683654e-05, 2.246501207991969e-05, 0.0017353435978293419, 0.0022565408144146204, 0.002218238776549697, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.99763023853302, 7.048921543173492e-05, 1.2420621260389453e-06, 6.923218461452052e-05, 0.0007476441096514463, 0.000763593299780041, 7.526726676587714e-06, 7.549879228463396e-05, 2.769195930341084e-07, 9.996434528147802e-05, 3.433594974922016e-05, 0.00019309873459860682, 0.00030683254590258, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9413580298423767, 0.0009730295860208571, 0.0004518784990068525, 0.0012102690525352955, 2.08603341889102e-06, 0.0018085278570652008, 1.965607407328207e-05, 0.03966164216399193, 0.0004176551301497966, 0.003035214263945818, 0.002604098990559578, 0.00038386619416996837, 0.007335478439927101, 0.0007384858326986432, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.98450767993927, 0.00010836195724550635, 2.4642587959533557e-05, 0.0004206285811960697, 1.2694325732809375e-06, 0.00011915803042938933, 9.773763304110616e-05, 0.0023661903105676174, 0.00036190610262565315, 0.003964339382946491, 0.0009553474374115467, 0.0006011889781802893, 0.002696628449484706, 0.00010731287329690531, 0.003667621174827218, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9236841797828674, 0.0004220438713673502, 1.1906979125342332e-05, 0.036959629505872726, 0.00012378330575302243, 0.0005373558960855007, 0.0029662158340215683, 0.0006545133655890822, 5.279582546791062e-05, 0.0077215940691530704, 0.0131848668679595, 7.147376891225576e-05, 0.00951047521084547, 0.0007938834605738521, 0.0006112216506153345, 0.0026941129472106695, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9707465171813965, 7.250557973748073e-05, 4.7150632553893956e-07, 0.0007667056052014232, 1.613832296243345e-06, 4.375730350147933e-05, 0.00013073488662485033, 0.0010341654997318983, 0.000541800691280514, 0.006641003768891096, 0.014592697843909264, 0.00018458777049090713, 0.003935785032808781, 3.716452511071111e-06, 2.8841026505688205e-05, 0.0008475544163957238, 0.0004274799139238894, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9573933482170105, 0.00019599858205765486, 2.3228804479913379e-07, 0.003389731515198946, 9.590152103555738e-07, 5.782760854344815e-05, 1.3954500900581479e-05, 9.882948506856337e-05, 0.00014849765284452587, 0.006945967674255371, 0.01562657207250595, 8.081036867224611e-06, 0.011182066984474659, 6.349627255985979e-06, 2.5934892619261518e-05, 0.00012999530008528382, 0.0007364018238149583, 0.004039338324218988, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.990619421005249, 6.466358172474429e-05, 9.901318662741687e-07, 0.00021274133177939802, 1.4796115465287585e-05, 0.00017918922821991146, 6.778116585337557e-06, 7.372434629360214e-05, 0.0002996290859300643, 0.0013087284751236439, 0.0005674119456671178, 0.0011630650842562318, 0.0028649545274674892, 9.635516471462324e-05, 8.433945913566276e-05, 0.00028768807533197105, 4.3501222535269335e-05, 0.00038704575854353607, 0.001724953530356288, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9956457018852234, 2.669214882189408e-05, 3.9460044121142346e-08, 0.001181368250399828, 1.471557879995089e-05, 0.0001445170200895518, 0.0004370114766061306, 3.213761010556482e-05, 8.180093402643251e-08, 9.59495737333782e-05, 0.0007029082044027746, 1.2536466783785727e-05, 0.00018710187578108162, 0.00011147692566737533, 7.006971713963139e-07, 4.411066583998036e-06, 0.0007388183148577809, 4.973216618964216e-06, 9.905266779242083e-05, 0.0005597113631665707, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9956768155097961, 3.464311521383934e-05, 5.037085770709382e-08, 0.00012908628559671342, 3.5288087474327767e-06, 8.07559808890801e-06, 3.11900585074909e-05, 6.715198833262548e-05, 1.5350008197856368e-07, 0.00010576295608188957, 0.0004207944730296731, 4.739598443848081e-05, 0.00017379986820742488, 3.910295163223054e-06, 3.7129705106053734e-06, 6.050187221262604e-05, 4.920005812891759e-05, 4.588661977322772e-05, 0.0002989978529512882, 0.00027882878202944994, 0.0025604318361729383, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9873839616775513, 0.002735754009336233, 1.0963850400003139e-05, 0.0005862272810190916, 6.101510280132061e-06, 6.327711162157357e-05, 2.1911782823735848e-05, 0.00033786968560889363, 6.588418182218447e-05, 0.0013113167369738221, 0.0019715256057679653, 0.00013529015996027738, 0.0018900128779932857, 0.00018075927800964564, 0.00026106255245395005, 0.00037814932875335217, 9.485983173362911e-05, 4.87527868244797e-05, 0.00035683103487826884, 0.0007243757136166096, 0.00010285121243214235, 0.0013322114245966077, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3808109164237976, 0.005228960886597633, 0.6019217371940613, 0.006041430402547121, 3.194508099113591e-05, 4.269973578630015e-05, 8.645826892461628e-05, 0.0006069993833079934, 0.0006299586966633797, 0.0009580003097653389, 0.0006117315497249365, 0.0002489396429155022, 0.0002464656427036971, 6.495700654340908e-05, 0.0006897134589962661, 0.00021990980894770473, 7.002806523814797e-05, 0.00014192514936439693, 2.700363074836787e-05, 5.169477663002908e-05, 4.5723249968432356e-06, 0.00029897037893533707, 0.0009650306892581284, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.563797116279602, 0.0085763493552804, 0.004927160684019327, 0.34193748235702515, 2.1189502149354666e-05, 0.00020753320131916553, 0.0003593186556827277, 0.0014688574010506272, 0.0003070282400585711, 0.02934207208454609, 0.004261575639247894, 8.383052772842348e-06, 0.0024304059334099293, 9.930188389262185e-05, 0.000542300462257117, 9.130241960519925e-05, 0.03218064829707146, 7.990232552401721e-05, 0.0003810952475760132, 0.00010837644367711619, 3.33801144734025e-05, 0.0008215161506086588, 0.0017809885321184993, 0.006236794404685497, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3740951418876648, 0.00012492266250774264, 8.869202247296926e-06, 0.0009476605337113142, 0.6135826706886292, 0.002516972366720438, 0.0002441502292640507, 5.642483665724285e-05, 1.795958496586536e-06, 0.0004439418844413012, 5.296186645864509e-05, 0.0001609584578545764, 0.0007701200665906072, 0.005056836176663637, 1.382048776576994e-05, 5.882297773496248e-05, 0.00018946161435451359, 0.00012655068712774664, 3.178522456437349e-05, 6.542137998621911e-05, 1.7726270016282797e-05, 0.0008768205880187452, 0.00038724290789105, 9.015519140120887e-07, 0.00016804093320388347, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.18932704627513885, 0.0003522285260260105, 1.1454378181952052e-05, 0.0007539974758401513, 0.003070831298828125, 0.7761431336402893, 5.7844576076604426e-05, 0.004086362663656473, 1.6754691387177445e-05, 0.0018927170895040035, 0.00031636765925213695, 0.0005395743064582348, 0.005218177568167448, 0.007962456904351711, 0.004062105901539326, 0.002489584032446146, 1.4082329471420962e-05, 1.4990718227636535e-05, 0.00015581185289192945, 0.0003996991435997188, 5.5086038628360257e-05, 0.0019400393357500434, 0.0003281844255980104, 1.4123389746600878e-07, 8.962309038906824e-06, 0.0007822647457942367, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02010822296142578, 2.635493547131773e-05, 2.7333503567206208e-06, 0.0002452948538120836, 1.4769997505936772e-05, 2.0045878045493737e-05, 0.9765581488609314, 0.0007374248816631734, 2.2332008029479766e-06, 0.00016837129078339785, 0.00137846521101892, 1.0426949756947579e-06, 9.273762407246977e-05, 6.001193469273858e-06, 1.852532113844063e-05, 7.490423013223335e-05, 2.892227712436579e-05, 4.0283404814545065e-05, 1.959171459020581e-05, 2.944982952612918e-05, 8.47621322463965e-06, 0.0002561432193033397, 5.941176277701743e-05, 2.014576523379219e-07, 7.743915921309963e-05, 7.494620604120428e-06, 1.74276083271252e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.22277945280075073, 0.002593980636447668, 2.9011016522417776e-06, 0.0005522919236682355, 1.904411533359962e-06, 0.0006727795698679984, 0.0021306686103343964, 0.6761497855186462, 6.977441080380231e-05, 0.003776289988309145, 0.06293372064828873, 0.0015865935711190104, 0.0017082190606743097, 3.881497832480818e-05, 0.009209989570081234, 0.0013166061835363507, 0.00018038915004581213, 0.001267515355721116, 0.0018657379550859332, 0.0009925103513523936, 3.7589063140330836e-05, 0.006426853593438864, 0.0020985312294214964, 3.3772406027310353e-07, 2.868737283279188e-05, 2.698123182653944e-07, 0.00025180247030220926, 0.001325962133705616, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08230321109294891, 0.0005252842092886567, 4.260616606188705e-06, 0.0006264003459364176, 8.583574384601889e-08, 4.224356871418422e-06, 4.716531293524895e-06, 0.0003961432375945151, 0.8586453795433044, 0.05151975154876709, 0.0026339711621403694, 1.6450698240078054e-05, 0.0006709631998091936, 1.5013217762316344e-06, 9.040276927407831e-05, 7.098149944795296e-05, 7.727313459326979e-06, 0.00020618800772354007, 0.0005295966984704137, 0.0013012847630307078, 5.2623075674773645e-08, 9.972530097002164e-05, 0.00018814747454598546, 2.380612386332359e-06, 6.255433254409581e-05, 3.8025717685741256e-08, 9.557845714880386e-07, 3.836729320028098e-06, 8.390715811401606e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.465234637260437, 0.004093279130756855, 6.743345238646725e-06, 0.004121774807572365, 5.443537247629138e-07, 5.419518129201606e-05, 9.294981850871409e-07, 0.00026017383788712323, 0.00907624140381813, 0.5008078813552856, 0.0023183668963611126, 2.4238510377472267e-05, 0.0030431346967816353, 4.68218931928277e-05, 0.00012087374489055946, 0.0001290314394282177, 0.00011785242531914264, 4.014383375761099e-05, 0.0024619202595204115, 0.0034230302553623915, 1.7263497511521564e-06, 0.0002791083534248173, 0.002281592693179846, 4.889936462859623e-06, 0.00025089550763368607, 2.8178769184705743e-07, 1.050695664162049e-05, 1.070987195816997e-06, 4.786956196767278e-05, 0.0017402973026037216, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3585144281387329, 0.0011027655564248562, 0.00016592990141361952, 0.013357667252421379, 2.546785253798589e-07, 6.185192614793777e-05, 0.0057292357087135315, 0.010611956007778645, 0.0007551729213446379, 0.06400889158248901, 0.4949103593826294, 8.391502342419699e-05, 0.005665580276399851, 2.122150362993125e-06, 0.0002814042672980577, 0.0004684626474045217, 0.0004158185620326549, 0.0012447141343727708, 0.012713258154690266, 0.0010433174902573228, 1.1126050594612025e-05, 0.0011420734226703644, 0.0006368785980157554, 7.042537617962807e-05, 0.002744977129623294, 1.2284903050385765e-07, 1.483806499891216e-05, 0.005110186990350485, 0.004603618290275335, 0.00015696643094997853, 0.014371593482792377, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.387024849653244, 0.0012950673699378967, 4.0595266909804195e-05, 0.0016053735744208097, 5.915556903346442e-05, 0.00023957251687534153, 2.72913839580724e-05, 0.004568422678858042, 0.0004761815653182566, 0.01562541350722313, 0.011085761711001396, 0.49522680044174194, 0.03870179131627083, 0.00026717083528637886, 0.0020083452109247446, 0.004042898770421743, 0.00032641590223647654, 0.0036371152382344007, 0.005066386889666319, 0.003996914252638817, 5.871349458175246e-06, 0.012405160814523697, 0.004702437669038773, 1.1574948075576685e-05, 0.0001442672364646569, 1.3832131116942037e-05, 7.560355879832059e-05, 1.5469962818315253e-05, 0.0020267819054424763, 3.810078851529397e-05, 0.003570300992578268, 0.001669161138124764, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4725484848022461, 0.0012542733456939459, 1.6021392639231635e-06, 0.007470476441085339, 2.098127151839435e-05, 0.00022495862504001707, 3.144747097394429e-06, 0.00015600249753333628, 5.6978915381478146e-05, 0.008746420964598656, 0.006501034367829561, 0.005835692398250103, 0.4674169719219208, 0.0013900346821174026, 0.0022286404855549335, 0.0015983363846316934, 0.0009148887475021183, 4.253075894666836e-05, 0.007089768536388874, 0.002937783719971776, 9.027125088323373e-06, 0.002526575466617942, 0.004196966998279095, 9.68642325460678e-07, 0.0005582970916293561, 1.858499126683455e-05, 8.944844012148678e-05, 6.846757969469763e-06, 7.68072932260111e-05, 1.4904732779541519e-05, 0.0030944147147238255, 0.0010494303423911333, 0.0019187491852790117, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14304865896701813, 0.0001688210031716153, 5.145543582329992e-07, 0.00013525491522159427, 0.001670985366217792, 0.000564366695471108, 4.049839844810776e-06, 4.1035786125576124e-05, 6.743645712958823e-07, 0.00026636559050530195, 3.54247895302251e-05, 0.00038403936196118593, 0.001104785711504519, 0.8484564423561096, 0.00022315295063890517, 0.00125994929112494, 2.182349817303475e-05, 4.0234138509731565e-07, 9.149295692623127e-06, 7.982308306964114e-05, 3.274861228419468e-05, 0.000818326196167618, 0.0002738100884016603, 6.036240307594198e-08, 5.658039754052879e-06, 0.0007130159065127373, 0.00026146037271246314, 3.5881919302482856e-06, 2.1928895876044407e-05, 4.159101507639207e-08, 0.00011863742111017928, 1.6391287545047817e-06, 3.8121448596939445e-05, 0.00023530120961368084, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.24328209459781647, 0.0009063901379704475, 0.0002717962197493762, 0.0012521950993686914, 3.105668383796001e-06, 0.001761557999998331, 7.013605227257358e-06, 0.040556132793426514, 0.0003538632008712739, 0.003045480465516448, 0.0016025315271690488, 0.0006016580737195909, 0.009119357913732529, 0.00086904939962551, 0.6830158829689026, 0.002626276109367609, 0.00010224885045317933, 7.45571151128388e-06, 3.991548510384746e-05, 0.00021531074889935553, 2.0861371012870222e-06, 0.000726094760466367, 0.0004009866388514638, 1.702951885818038e-05, 6.652241427218542e-05, 3.073251377827546e-07, 0.00018005711899604648, 5.3525491239270195e-06, 0.0071047646924853325, 2.1172911147004925e-05, 0.00044749100925400853, 5.941993367741816e-05, 3.588023173506372e-05, 0.001178023056127131, 0.00011552044452400878, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.24815866351127625, 0.0004487273108679801, 9.697681889520027e-06, 0.0010719134006649256, 7.626599654031452e-06, 0.00018865316815208644, 6.339164974633604e-05, 0.004560624714940786, 0.0010741765145212412, 0.015649856999516487, 0.002170345513150096, 0.0007331715896725655, 0.007097125519067049, 0.00044510516454465687, 0.016780385747551918, 0.6907860040664673, 0.00019302917644381523, 0.0003835348761640489, 0.0009248864953406155, 0.001862714416347444, 2.057407328948102e-07, 0.002524268114939332, 0.0015590769471600652, 3.3248329600610305e-07, 1.708475247141905e-05, 4.2712878212114447e-07, 1.1026169886463322e-05, 4.378564426588127e-06, 0.00015843719302210957, 1.5476502085221e-05, 0.0009950557723641396, 4.288491982151754e-05, 5.0623875722521916e-05, 0.000995215610601008, 1.3690095329366159e-05, 0.0010022438364103436, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15202470123767853, 0.0005997071275487542, 1.1835468285426032e-05, 0.10644718259572983, 9.087667422136292e-05, 0.0003228001296520233, 0.000542075838893652, 0.0006674103206023574, 0.00022456956503447145, 0.021209707483649254, 0.019741186872124672, 0.00012132841220591217, 0.015023781917989254, 0.0003238526696804911, 0.0003435326216276735, 0.003281805431470275, 0.6119738221168518, 0.00224235188215971, 0.008034387603402138, 0.0039560976438224316, 0.00143277901224792, 0.011238718405365944, 0.0009119117166846991, 1.6121168300742283e-05, 0.034571193158626556, 4.049971539643593e-05, 7.184680725913495e-05, 0.0005501421983353794, 0.0001563764235470444, 6.361229679896496e-06, 0.00147312565241009, 0.0009954210836440325, 1.2639467058761511e-05, 0.0010144341504201293, 3.702372487168759e-05, 5.788439739262685e-05, 0.00023058420629240572, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.035443101078271866, 2.0004812540719286e-05, 4.18805292667912e-08, 0.0003809734480455518, 5.874024395779998e-07, 2.792186933220364e-06, 2.425247657811269e-05, 8.103247819235548e-05, 0.00011144909512950107, 0.0027193769346922636, 0.00454584788531065, 4.083846943103708e-05, 0.0014098533429205418, 5.68071186535235e-07, 3.1567922178510344e-06, 8.66913323989138e-05, 0.0001365105708828196, 0.8487799763679504, 0.10322029143571854, 0.00040226749842986465, 2.244725919808843e-06, 0.0014588433550670743, 0.00012918819265905768, 6.671142216418957e-08, 7.290130452020094e-05, 3.444819185460801e-07, 1.4172115925248363e-06, 1.9379569494049065e-05, 4.5301294449018314e-05, 6.376938017638167e-06, 0.00026465157861821353, 0.0004582676920108497, 2.116967152687721e-05, 9.94484216789715e-05, 4.348984106172793e-08, 3.564729809113487e-07, 2.450852434776607e-06, 8.022138899832498e-06, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2318733036518097, 0.0003618353803176433, 8.543166529761947e-08, 0.006350396666675806, 1.0386469284640043e-06, 1.706758121144958e-05, 9.30641544982791e-06, 0.00014515469956677407, 9.0070599981118e-05, 0.01544187217950821, 0.04194100946187973, 8.23421487439191e-06, 0.022686535492539406, 5.4958513828751165e-06, 2.1405727238743566e-05, 5.323981531546451e-05, 0.0014441021485254169, 0.005718633532524109, 0.6479307413101196, 0.012128104455769062, 4.028283728985116e-05, 0.002520335605368018, 0.0007621925324201584, 6.570355992607801e-08, 0.0007668299949727952, 4.547823095890635e-07, 8.124479791149497e-06, 1.3828406736138277e-05, 3.526886212057434e-05, 9.165401934296824e-06, 0.0020918850786983967, 0.004286687355488539, 3.2024597658164566e-06, 0.0017294595018029213, 5.357091481528187e-07, 2.501844164726208e-06, 2.884956984416931e-06, 8.676738070789725e-05, 0.0014117915416136384, 0.0, 0.0, 0.0, 0.0], [0.3010721206665039, 0.0003067184588871896, 7.214547395051341e-07, 0.0010787947103381157, 3.671133526950143e-05, 9.958079317584634e-05, 2.7117800982523477e-06, 8.51582590257749e-05, 0.0006431526271626353, 0.00647402461618185, 0.0024339223746210337, 0.0026007171254605055, 0.012783723883330822, 0.0001047167825163342, 7.190534233814105e-05, 0.00023829020210541785, 0.00014132891374174505, 0.0010901358909904957, 0.006501613184809685, 0.6443015933036804, 0.0005654482520185411, 0.011171936988830566, 0.0017330633709207177, 3.266845851612743e-07, 7.267465116456151e-05, 1.726137998048216e-05, 5.252150003798306e-05, 2.8104516331950435e-06, 2.3941935069160536e-05, 6.633036537095904e-05, 0.002350717782974243, 0.00014581516734324396, 0.0007032529101707041, 0.0013972821179777384, 3.202367588528432e-05, 2.3009477445157245e-05, 2.68677049461985e-05, 1.7945263607543893e-05, 0.0001209545589517802, 0.001408109557814896, 0.0, 0.0, 0.0], [0.012865342199802399, 1.5182659808488097e-05, 2.972013524171757e-09, 0.0002875621721614152, 1.2528023489721818e-06, 1.252613128599478e-05, 4.250307028996758e-06, 8.040903594519477e-06, 3.2277103656497275e-08, 5.239708480075933e-05, 0.00013598549412563443, 1.002539192995755e-06, 5.5890355724841356e-05, 7.012691639829427e-06, 9.724845284608818e-08, 3.212419130704802e-07, 4.670393173000775e-05, 1.5471712231374113e-06, 1.7591864889254794e-05, 5.334270463208668e-05, 0.9805435538291931, 0.005806764122098684, 1.860846532508731e-05, 1.2037257857144823e-09, 3.0451696147792973e-05, 6.754650598850276e-07, 4.56516090707737e-06, 5.748487637902144e-06, 3.3398507639503805e-06, 7.534131096775809e-10, 7.689136509725358e-06, 3.901714080711827e-06, 4.307142731363456e-08, 5.192545700083429e-07, 5.106750791128434e-07, 8.534786211100709e-09, 1.0071753209217604e-08, 2.59315993389464e-06, 1.4836371065030107e-07, 3.5096545047963446e-07, 4.402238118927926e-06, 0.0, 0.0], [0.2408866286277771, 0.0003843706217594445, 3.8272382596460375e-08, 0.0007521198713220656, 4.429959517437965e-06, 1.3616883734357543e-05, 1.0370951713412069e-05, 0.00026089049060828984, 2.180447154387366e-06, 0.0008239771705120802, 0.0025784452445805073, 0.0001127499490394257, 0.001231508213095367, 3.884355010086438e-06, 3.183517401339486e-05, 0.00015905841428320855, 0.00016030347615014762, 0.00039731874130666256, 0.0017629768699407578, 0.001543086371384561, 0.0018108984222635627, 0.7393378019332886, 0.00752194132655859, 2.4824957645108725e-09, 2.4742143068579026e-05, 3.882949215494591e-07, 2.5054800971702207e-06, 3.208101816198905e-06, 4.296221959521063e-05, 2.8116325268001674e-08, 1.3948349078418687e-05, 1.6366040654247627e-05, 6.464315902121598e-06, 3.004602695000358e-05, 1.3071932869479497e-07, 9.608824029783136e-07, 1.787801124919497e-06, 1.753078095134697e-06, 4.386996351968264e-06, 1.6937889085966162e-05, 1.7175851098727435e-05, 2.5725426894496195e-05, 0.0], [0.5535919070243835, 0.17262260615825653, 5.0000453484244645e-05, 0.01042777020484209, 8.65629335748963e-05, 0.0002456592337694019, 5.955337837804109e-05, 0.001045247190631926, 0.0008384220418520272, 0.01357327215373516, 0.011254074051976204, 0.003105845768004656, 0.02095966413617134, 0.0010894862934947014, 0.0004093680181540549, 0.002362574217841029, 0.0016563499812036753, 0.0009357953676953912, 0.0033135719131678343, 0.00543051166459918, 0.0010150577872991562, 0.032745711505413055, 0.1570950448513031, 3.374255175003782e-05, 0.0007222488638944924, 2.940950980701018e-05, 6.13709562458098e-05, 3.093114719376899e-05, 0.00024038334959186614, 4.459061528905295e-05, 0.0006923413602635264, 0.00030414367211051285, 0.0001727685157675296, 0.0012000406859442592, 0.00010696896788431332, 2.8368263883749023e-05, 1.842521487560589e-05, 2.8223597837495618e-05, 3.744996865862049e-05, 0.00011934195208596066, 0.00018961371097248048, 3.498619480524212e-05, 0.0019905013032257557]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9940896034240723, 0.0059104301035404205, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9918997287750244, 0.002507372060790658, 0.005592965055257082, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.99346524477005, 0.003521271515637636, 0.00027059370768256485, 0.0027428686153143644, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9289718270301819, 0.003740765620023012, 0.0001203682622872293, 0.002464901190251112, 0.06470214575529099, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9885430335998535, 0.0006141006015241146, 0.00041145377326756716, 0.007392834406346083, 0.000539508939255029, 0.002499047899618745, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9694589376449585, 0.0009395341039635241, 2.8261547413421795e-05, 0.0020353952422738075, 2.71175304078497e-05, 0.00829475186765194, 0.019215956330299377, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9474747776985168, 0.005424594972282648, 0.004983103834092617, 0.021659083664417267, 1.2853307453042362e-05, 0.0011076473165303469, 0.001525420811958611, 0.01781250163912773, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9157270193099976, 0.006453890353441238, 0.0011921703116968274, 0.04077358543872833, 3.2812098652357236e-05, 0.002033841796219349, 0.00022136607731226832, 0.010999627411365509, 0.022565681487321854, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8299553990364075, 0.0021487579215317965, 0.002185288118198514, 0.04418088495731354, 1.725433139654342e-05, 0.0014081649715080857, 0.020800573751330376, 0.08762412518262863, 0.000955707801040262, 0.010723880492150784, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9581490159034729, 0.004779844544827938, 0.00035515165654942393, 0.0029918320942670107, 0.0032907268032431602, 0.007958430796861649, 0.0017856257036328316, 0.013284464366734028, 0.00015784945571795106, 0.0009225388057529926, 0.0063244919292628765, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.944205641746521, 0.0028841965831816196, 4.829783574678004e-05, 0.007261567283421755, 0.000468916492536664, 0.005381070543080568, 0.00017496546206530184, 0.006957130506634712, 0.00012532716209534556, 0.004074090626090765, 0.016885550692677498, 0.011533298529684544, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7170465588569641, 0.0029263142496347427, 1.372535371046979e-05, 0.0010318169370293617, 0.1216491237282753, 0.14445394277572632, 0.0009355274378322065, 0.005332366097718477, 4.7773132791917305e-06, 0.0012480594450607896, 0.00030511102522723377, 0.0019744164310395718, 0.0030781622044742107, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7894357442855835, 0.0020973754581063986, 0.0020318119786679745, 0.010099953971803188, 0.00036911203642375767, 0.06713488698005676, 0.001530290232039988, 0.08991599082946777, 0.0002695393341127783, 0.005345543846487999, 0.005062697920948267, 0.0006220968207344413, 0.019640257582068443, 0.006444734986871481, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8822957873344421, 0.0018603845965117216, 0.00040732327033765614, 0.003660863032564521, 0.0005736579769290984, 0.007988521829247475, 0.02257135882973671, 0.02860121801495552, 0.0003292130131740123, 0.0042566233314573765, 0.006161138415336609, 0.0010443199425935745, 0.02063308097422123, 0.005851183086633682, 0.013765350915491581, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.786965548992157, 0.0042131864465773106, 0.0001677500840742141, 0.06983140110969543, 0.002818087814375758, 0.007715716026723385, 0.031199535354971886, 0.017051437869668007, 0.00018332427134737372, 0.012005697004497051, 0.017662225291132927, 0.0002890100877266377, 0.03032497689127922, 0.004889386706054211, 0.003095801919698715, 0.011587009765207767, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9286779761314392, 0.0015812769997864962, 1.4040829228179064e-06, 0.0019428395899012685, 1.1035229363187682e-05, 0.0002707966777961701, 0.002038027858361602, 0.0036648765671998262, 0.00022354313114192337, 0.0007879888289608061, 0.04505370557308197, 0.00048093145596794784, 0.010849039070308208, 3.6126039049122483e-06, 6.594638398382813e-05, 0.0008642339380457997, 0.0034828479401767254, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8384721875190735, 0.0026208001654595137, 3.342468335176818e-05, 0.01232653483748436, 7.991644088178873e-05, 0.003431107848882675, 0.0022304044105112553, 0.008115601725876331, 0.0006596868042834103, 0.008109694346785545, 0.0638236477971077, 0.00041881605284288526, 0.03681996464729309, 0.00012104758206987754, 0.0005988333723507822, 0.00862804614007473, 0.003624672070145607, 0.00988560076802969, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8221591114997864, 0.0057971584610641, 0.00019727385370060802, 0.005305740050971508, 0.0009490191005170345, 0.01202884316444397, 0.00190554594155401, 0.0023991470225155354, 0.01059637125581503, 0.014539752155542374, 0.011935291811823845, 0.01170417945832014, 0.048201464116573334, 0.006271726917475462, 0.006691561546176672, 0.02466079778969288, 0.005656811874359846, 0.005013483576476574, 0.003986795898526907, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9537661671638489, 0.0015298646176233888, 5.0040102905768435e-06, 0.007204053923487663, 0.0002875779173336923, 0.0017434227047488093, 0.00982952956110239, 0.004322257358580828, 3.13615760205721e-06, 0.00010985424887621775, 0.008465062826871872, 0.0003484692715574056, 0.0013010905822739005, 0.0008953196229413152, 6.606405804632232e-05, 5.8244528190698475e-05, 0.0037666442804038525, 0.0005685480427928269, 0.001456835656426847, 0.004272699821740389, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9650986790657043, 0.0014334715669974685, 4.691014055424603e-06, 0.0012125533539801836, 0.0003355584340170026, 0.0012382488930597901, 0.0013702923897653818, 0.002564979949966073, 6.760245128134557e-07, 7.471856224583462e-05, 0.001875573769211769, 0.0005754472804255784, 0.0028297884855419397, 0.0007055316236801445, 0.00027714675525203347, 0.00048612672253511846, 0.001569450250826776, 0.0003992257115896791, 0.0028444859199225903, 0.004788658116012812, 0.010314569808542728, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8593109250068665, 0.020267227664589882, 0.002777141286060214, 0.013781089335680008, 0.00030035056988708675, 0.002284612273797393, 0.0010426462395116687, 0.009922247380018234, 0.003133647609502077, 0.007885495200753212, 0.009642801247537136, 0.001854671980254352, 0.00787472166121006, 0.0011870941380038857, 0.002824161434546113, 0.00242280843667686, 0.004221721086651087, 0.004099525045603514, 0.006750837899744511, 0.0058750067837536335, 0.002170956926420331, 0.030370241031050682, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10544665902853012, 0.01270485483109951, 0.7639007568359375, 0.08842571824789047, 0.00018797894881572574, 0.0002628951915539801, 0.0007833988056518137, 0.001709726988337934, 0.002409537322819233, 0.014530741609632969, 0.0006739442469552159, 3.582143835956231e-05, 0.0008068872266449034, 0.000150876454426907, 0.001663832925260067, 0.0002909536415245384, 0.001969669247046113, 2.807584314723499e-05, 8.648991206428036e-05, 0.0001831684639910236, 5.4813881433801726e-05, 0.0004074028111062944, 0.003285784274339676, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.33122679591178894, 0.007398602087050676, 0.006046590860933065, 0.38090354204177856, 0.0007559282821603119, 0.003547786269336939, 0.00850890576839447, 0.02200973406434059, 0.0006401475402526557, 0.06486588716506958, 0.012087974697351456, 2.1607689632219262e-05, 0.004766723141074181, 0.0004923217929899693, 0.002488807076588273, 0.00041253515519201756, 0.11648410558700562, 0.00017437727365177125, 0.001129980431869626, 0.0012968714581802487, 0.001893762033432722, 0.006374640390276909, 0.012117794714868069, 0.014354660175740719, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.022601181641221046, 0.0004386410873848945, 2.7166800009581493e-06, 0.0003694462648127228, 0.9364854693412781, 0.032159216701984406, 0.00016640119429212064, 0.00037730683106929064, 2.719356189118116e-07, 9.796216181712225e-05, 2.4832312192302197e-05, 0.00014765629020985216, 0.0004301169828977436, 0.004391680471599102, 0.0003911138919647783, 5.064404831500724e-05, 0.00018704244575928897, 6.401351129170507e-06, 2.0583942387020215e-05, 0.0002504437288735062, 4.058926060679369e-05, 0.0006710129673592746, 0.0006265917327255011, 2.0016514099552296e-06, 6.063750697649084e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.039748288691043854, 0.0016230319160968065, 2.9864083899155958e-06, 0.0004959744983352721, 0.0075991530902683735, 0.903511106967926, 0.00030855785007588565, 0.020541049540042877, 3.7260215322021395e-06, 0.0009059454896487296, 0.0005234951386228204, 0.00019036592857446522, 0.002769604790955782, 0.001758625265210867, 0.006682183127850294, 0.0016490903217345476, 0.00027723185485228896, 2.3619986677658744e-05, 0.0003351525228936225, 0.001790589652955532, 0.00015660120698157698, 0.0019276170060038567, 0.0019798888824880123, 1.2426404509824351e-06, 5.812747258460149e-05, 0.005136688239872456, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.017394619062542915, 8.34300517453812e-05, 1.299515815844643e-06, 0.0006728374864906073, 1.4733542229805607e-05, 0.00010638036474119872, 0.9528148770332336, 0.021972037851810455, 7.827152899153589e-07, 1.5493780665565282e-05, 0.0031969614792615175, 3.257389471400529e-05, 0.0001382468908559531, 2.389342444075737e-05, 0.000840842432808131, 0.00010894631850533187, 0.00028938756440766156, 0.0001089509969460778, 6.79441072861664e-05, 0.00012526210048235953, 0.00019726519531104714, 0.0008631302043795586, 0.0002980587596539408, 2.3374896045424975e-06, 0.0003466096241027117, 2.3777269234415144e-05, 0.00025927400565706193, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05558102950453758, 0.0006445589242503047, 4.0934705225481594e-07, 0.00042194841080345213, 1.340792550763581e-06, 0.000646521570160985, 0.0018358062952756882, 0.9107965230941772, 6.180139735079138e-06, 0.00014045907300896943, 0.010802187956869602, 0.0002091953792842105, 0.0011495650978758931, 1.2848360711359419e-05, 0.003265448845922947, 0.0001199487887788564, 0.00020614646200556308, 0.0006146047380752861, 0.0012190378038212657, 0.001267205341719091, 0.00038551114266738296, 0.0035668814089149237, 0.0011672672117128968, 5.425858375929238e-07, 0.0003176873433403671, 1.1132586905659991e-06, 0.002803834853693843, 0.002816123189404607, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08360803127288818, 0.0049411761574447155, 5.052056076237932e-05, 0.007228298112750053, 1.1925421858904883e-06, 0.000162318188813515, 0.0002728418621700257, 0.005935708060860634, 0.6047206521034241, 0.26288917660713196, 0.005585616920143366, 7.11743050487712e-05, 0.0013994898181408644, 2.673692279131501e-06, 0.0015110736712813377, 9.4614464615006e-05, 0.0003633210726547986, 0.0007099778158590198, 0.0009741897229105234, 0.003578288946300745, 1.0970221410389058e-05, 0.0001563943224027753, 0.003689345670863986, 0.0002846125280484557, 0.004334022291004658, 1.3892353081246256e-06, 0.00024127897631842643, 0.0001852042623795569, 0.006996475625783205, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1152101382613182, 0.006714800372719765, 0.000162112366524525, 0.0255129411816597, 6.206613761605695e-06, 0.0004051269206684083, 5.4560387070523575e-05, 0.00351078063249588, 0.014202400110661983, 0.7775508165359497, 0.0034905581269413233, 5.7819983339868486e-05, 0.0047684297896921635, 3.606372411013581e-05, 0.0007714288076385856, 0.0003429505741223693, 0.002843409776687622, 9.518005390418693e-05, 0.003136677900329232, 0.005728919059038162, 4.441967757884413e-05, 0.0004939687205478549, 0.008516553789377213, 0.0005445042625069618, 0.014522203244268894, 5.894724836252863e-06, 0.00030799367232248187, 2.679031968000345e-05, 0.0010097086196765304, 0.009926472790539265, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.03713580593466759, 0.0008775312453508377, 7.848548557376489e-05, 0.015299011021852493, 1.4544053783538402e-06, 0.00010877755266847089, 0.006165174767374992, 0.014768626540899277, 7.875503069953993e-05, 0.001974995480850339, 0.845594584941864, 7.480713975382969e-05, 0.0028255584184080362, 5.115834369462391e-07, 0.00017026669229380786, 0.00015173685096669942, 0.006004323251545429, 0.0034732050262391567, 0.012612894177436829, 0.0006110360845923424, 0.0005419608787633479, 0.0009975193534046412, 0.002906821435317397, 0.0003956863656640053, 0.020268796011805534, 1.613207473383227e-06, 0.00021117184951435775, 0.007802305277436972, 0.017504550516605377, 2.8824499167967588e-05, 0.0013330691726878285, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04171165078878403, 0.0007273957598954439, 5.084281838207971e-06, 0.0004205020668450743, 0.0002682959020603448, 0.00036437364178709686, 0.00021653271687682718, 0.0014350030105561018, 5.140026132721687e-06, 9.150908590527251e-05, 0.0012519678566604853, 0.9164962768554688, 0.025602256879210472, 0.0013075522147119045, 0.0030348259024322033, 0.0005533328512683511, 0.00034456001594662666, 0.00023525662254542112, 0.0004293296951800585, 0.00137053313665092, 3.670702062663622e-05, 0.0015910869697108865, 0.00048218751908279955, 9.56441908783745e-06, 0.00015524028276558965, 0.00017542645218782127, 0.0004897234030067921, 6.953987031010911e-05, 0.0008855744963511825, 1.1304659892630298e-06, 4.837113374378532e-05, 0.0001840855402406305, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1554105579853058, 0.0020603404846042395, 5.904707904846873e-06, 0.005144172348082066, 0.00025823267060332, 0.0014501075493171811, 5.391828017309308e-05, 0.0021640402264893055, 2.1527968783630058e-05, 0.0016064351657405496, 0.010191709734499454, 0.010151085443794727, 0.7491754293441772, 0.0027421838603913784, 0.01017574779689312, 0.005842445883899927, 0.0030403982382267714, 3.083394767600112e-05, 0.006112964358180761, 0.016732651740312576, 0.00015879959391895682, 0.0020674674306064844, 0.0016374929109588265, 1.1047049156331923e-05, 0.0014025868149474263, 0.0001992537872865796, 0.001220017671585083, 2.1487703634193167e-05, 0.0007244196604005992, 5.2953896556573454e-06, 0.0011675384594127536, 0.0018881573341786861, 0.0071258689276874065, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.03857047110795975, 0.00048676287406124175, 8.812001084379517e-08, 0.000122779980301857, 0.005581730045378208, 0.0025055643636733294, 2.1938227291684598e-05, 0.00011670660751406103, 8.340171575582644e-08, 6.456951086875051e-05, 1.648088073125109e-05, 0.00033052131766453385, 0.0009664044482633471, 0.9410394430160522, 0.0025494787842035294, 0.000986628350801766, 0.00023048926959745586, 2.1702271624235436e-07, 9.28598819882609e-06, 0.001075997482985258, 8.328325202455744e-05, 0.0006203338853083551, 0.0003860929573420435, 1.7099183935442852e-07, 1.0056071005237754e-05, 0.0019989400170743465, 0.0020207997877150774, 4.052772510476643e-06, 1.9309491108288057e-05, 3.980539009518225e-09, 1.3065056009509135e-05, 4.567590394799481e-07, 5.0416052545187995e-05, 0.00011722340423148125, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.32133859395980835, 0.0015032502124086022, 0.00035717309219762683, 0.009410788305103779, 0.00016890434199012816, 0.016028327867388725, 0.0002053897042060271, 0.019911326467990875, 6.106824002927169e-05, 0.0025995566975325346, 0.0018900673603639007, 0.0003402142319828272, 0.018113745376467705, 0.0032540573738515377, 0.5562437772750854, 0.008188020437955856, 0.0034578992053866386, 2.759976268862374e-05, 0.00021329266019165516, 0.004111619666218758, 7.46081059332937e-05, 0.0024103508330881596, 0.0031445701606571674, 0.0002873913326766342, 0.0028005591593682766, 5.076818706584163e-05, 0.010558384470641613, 8.335074380738661e-05, 0.007083627860993147, 1.715210783004295e-05, 0.0007566043641418219, 0.00013426884834188968, 4.393113704281859e-05, 0.0036519141867756844, 0.0014778183540329337, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.16036191582679749, 0.0012780018150806427, 2.2185913621797226e-05, 0.0023713421542197466, 0.0002248762029921636, 0.001783479005098343, 0.003998146392405033, 0.00463333074003458, 4.07644038205035e-05, 0.0015416644746437669, 0.0015637290198355913, 0.00047258863924071193, 0.01683974452316761, 0.0034527594689279795, 0.017767908051609993, 0.7616898417472839, 0.004626939073204994, 0.00039177873986773193, 0.000968358654063195, 0.0050940741784870625, 3.4372260415693745e-06, 0.0016141458181664348, 0.0016999210929498076, 1.1049841305066366e-05, 0.00029084301786497235, 4.1652379877632484e-05, 0.000830529781524092, 0.00043556030141189694, 0.0004473649896681309, 1.989761585718952e-06, 0.0003503866319078952, 6.375126395141706e-05, 6.234755710465834e-05, 0.0025966004468500614, 0.0005663724150508642, 0.00186053360812366, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.16436104476451874, 0.00263788178563118, 4.14170281146653e-05, 0.06607695668935776, 0.0011643714969977736, 0.0015624640509486198, 0.01255388930439949, 0.0027841543778777122, 3.6274232115829363e-05, 0.004333572927862406, 0.006344497669488192, 0.00013702016440220177, 0.022220438346266747, 0.001139097730629146, 0.0015757762594148517, 0.003894534194841981, 0.5750948190689087, 0.0019101364305242896, 0.004216517321765423, 0.005103054456412792, 0.0011178136337548494, 0.011998940259218216, 0.006141388788819313, 8.207879727706313e-05, 0.0696118101477623, 0.0019772748928517103, 0.0024057570844888687, 0.01315633486956358, 0.0013538515195250511, 9.255067197955213e-06, 0.003937660250812769, 0.0014234132831916213, 5.669038000633009e-05, 0.005460998509079218, 0.0004025489615742117, 0.0006155924638733268, 0.003060702932998538, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07928115874528885, 0.0006841905415058136, 6.815574948859648e-08, 0.0010951507138088346, 2.0406362182257e-06, 2.2378419089363888e-05, 0.0006314062047749758, 0.0003747308801393956, 2.3558166503789835e-05, 0.00018500992155168205, 0.008331519551575184, 7.647382153663784e-05, 0.00418297341093421, 2.444711526550236e-07, 1.1477874068077654e-05, 5.1371527661103755e-05, 0.0018863201839849353, 0.6889944076538086, 0.1946372091770172, 0.004766413476318121, 4.9866932386066765e-05, 0.0028069806285202503, 0.005558052100241184, 3.95493913174505e-07, 0.0008065987494774163, 3.353459078425658e-06, 5.2719624363817275e-05, 0.0003899818693753332, 0.0004888054681941867, 7.204272606031736e-06, 9.927168866852298e-05, 0.003168433904647827, 3.9500540879089385e-05, 0.0009883888997137547, 7.518784883586704e-08, 1.4315326097857906e-06, 6.484955520136282e-06, 0.00029431298025883734, 0.0, 0.0, 0.0, 0.0, 0.0], [0.28373828530311584, 0.004487877711653709, 5.058388069301145e-06, 0.019317252561450005, 5.4460702813230455e-05, 0.0012376051163300872, 0.0017604499589651823, 0.0063126953318715096, 0.00024137768195942044, 0.006789366248995066, 0.07160750776529312, 0.0003089959791395813, 0.06864359229803085, 4.332327443989925e-05, 0.0004192555497866124, 0.0020601244177669287, 0.006795847322791815, 0.01733478344976902, 0.33858922123908997, 0.03910181671380997, 0.0007660706178285182, 0.009993083775043488, 0.007829555310308933, 1.3115719411871396e-05, 0.01356492843478918, 6.270289304666221e-05, 0.0021037741098552942, 0.0009936662390828133, 0.00333457812666893, 9.826785390032455e-05, 0.006462769117206335, 0.03356613591313362, 0.00019831513054668903, 0.030609816312789917, 1.5713876564404927e-05, 0.0001204574218718335, 0.0005064337747171521, 0.001992761390283704, 0.018918951973319054, 0.0, 0.0, 0.0, 0.0], [0.25413063168525696, 0.007289606146514416, 1.6370242519769818e-05, 0.005644946824759245, 0.00047911194269545376, 0.003032122040167451, 0.0008039328386075795, 0.0005737085011787713, 0.001770934322848916, 0.0052279396913945675, 0.006573305930942297, 0.0037488588131964207, 0.039559945464134216, 0.0013069461565464735, 0.002049593720585108, 0.0032922588288784027, 0.005070534534752369, 0.0043929279781877995, 0.0029604455921798944, 0.5652295351028442, 0.006637280806899071, 0.020123612135648727, 0.013719636015594006, 3.509566158754751e-05, 0.00249256263487041, 0.0006804370786994696, 0.0050042420625686646, 0.0004963083192706108, 0.0002992783847730607, 0.0005683296476490796, 0.006156585179269314, 0.0014163292944431305, 0.002080241683870554, 0.01701081357896328, 0.0011587258195504546, 0.0008447651634924114, 0.0012247745180502534, 0.002057015197351575, 0.0027713908348232508, 0.0020689249504357576, 0.0, 0.0, 0.0], [0.02277885377407074, 0.00018681293295230716, 1.6159448890107342e-08, 0.0006753625930286944, 3.9720243876217864e-06, 1.8965230992762372e-05, 0.00014574530359823257, 0.00015363575948867947, 1.437089469646935e-08, 3.714960257639177e-06, 0.00038176553789526224, 9.189574484480545e-06, 0.00013344796025194228, 1.550033084640745e-05, 4.871678356721532e-06, 4.94858227284567e-07, 0.0004079081991221756, 1.881936623249203e-05, 3.495846613077447e-05, 0.00018379271205049008, 0.8935850262641907, 0.08001779019832611, 0.0004949963185936213, 6.95558668439844e-08, 0.0003611761494539678, 4.126275143789826e-06, 4.2135678086197004e-05, 0.0001535058399895206, 9.319555101683363e-05, 8.534826179129595e-10, 9.503900741947291e-07, 2.6511430405662395e-05, 9.136761605077481e-07, 4.42588907390018e-06, 3.6229419038136257e-06, 4.963176820638182e-07, 6.715119571509831e-09, 3.286409264546819e-05, 3.7280919968907256e-06, 3.7352272101998096e-06, 1.2850606253778096e-05, 0.0, 0.0], [0.04306096211075783, 0.00043120133341290057, 4.85736819655358e-08, 0.00029497192008420825, 6.446816769312136e-06, 1.794057789084036e-05, 5.2368359320098534e-05, 0.00012123115448048338, 8.686286356862638e-09, 4.085900854988722e-06, 0.00020099191169720143, 2.1525078409467824e-05, 0.0004682287108153105, 1.1544256267370656e-05, 3.74926021322608e-05, 7.720605935901403e-06, 0.0005387069541029632, 8.118282858049497e-05, 0.0002466407313477248, 0.0005759781342931092, 0.0023978566750884056, 0.9488213062286377, 0.002120599150657654, 3.640034407226267e-08, 7.682813156861812e-05, 2.893220880650915e-06, 1.444738245481858e-05, 1.2731605238514021e-05, 2.7991360184387304e-05, 3.7476780012646316e-10, 5.714333610740141e-07, 1.0850835678866133e-05, 3.7292174965841696e-06, 4.0900900785345584e-05, 3.826700321951648e-06, 6.126327207311988e-06, 5.169480346012278e-07, 4.086086119059473e-05, 1.2315524145378731e-05, 2.700609366002027e-05, 3.1881831091595814e-05, 0.00016747090558055788, 0.0], [0.26228708028793335, 0.04815318435430527, 0.0006689673755317926, 0.021233247593045235, 9.166418021777645e-05, 0.00039222234045155346, 0.00020396779291331768, 0.0021344097331166267, 0.001094849780201912, 0.004910758230835199, 0.006146768573671579, 0.0005123682785779238, 0.009844929911196232, 0.0001430303673259914, 0.0010762682650238276, 0.00034581031650304794, 0.010050452314317226, 0.008435730822384357, 0.008105281740427017, 0.00602466706186533, 0.0038135156501084566, 0.04830698296427727, 0.49913153052330017, 0.0012536782305687666, 0.033902086317539215, 0.00010517781629459932, 0.00037178199272602797, 8.200502634281293e-05, 0.0012060013832524419, 0.0001819609315134585, 0.0013505015522241592, 0.0009115547291003168, 0.00011081639240728691, 0.0016774724936112761, 3.677561107906513e-05, 0.00017677698633633554, 4.57889873359818e-05, 0.00101330759935081, 0.0026200786232948303, 0.002004811307415366, 0.0010430817492306232, 0.00036673821159638464, 0.008431942202150822]]], \"attentionHeadNames\": [\"L6H9\", \"L5H5\"], \"tokens\": [\"<|endoftext|>\", \"Research\", \" in\", \" mechan\", \"istic\", \" interpret\", \"ability\", \" seeks\", \" to\", \" explain\", \" behaviors\", \" of\", \" machine\", \" learning\", \" models\", \" in\", \" terms\", \" of\", \" their\", \" internal\", \" components\", \".\", \"Research\", \" in\", \" mechan\", \"istic\", \" interpret\", \"ability\", \" seeks\", \" to\", \" explain\", \" behaviors\", \" of\", \" machine\", \" learning\", \" models\", \" in\", \" terms\", \" of\", \" their\", \" internal\", \" components\", \".\"]}\n",
        "    )\n",
@@ -15001,10 +15001,10 @@
    "execution_count": 36,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:13.796662Z",
-     "iopub.status.busy": "2024-12-14T01:01:13.796322Z",
-     "iopub.status.idle": "2024-12-14T01:01:14.592546Z",
-     "shell.execute_reply": "2024-12-14T01:01:14.592025Z"
+     "iopub.execute_input": "2024-12-31T02:20:31.216210Z",
+     "iopub.status.busy": "2024-12-31T02:20:31.215816Z",
+     "iopub.status.idle": "2024-12-31T02:20:31.967447Z",
+     "shell.execute_reply": "2024-12-31T02:20:31.966868Z"
     }
    },
    "outputs": [
@@ -15102,10 +15102,10 @@
    "execution_count": 37,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:14.594877Z",
-     "iopub.status.busy": "2024-12-14T01:01:14.594530Z",
-     "iopub.status.idle": "2024-12-14T01:01:14.678853Z",
-     "shell.execute_reply": "2024-12-14T01:01:14.678279Z"
+     "iopub.execute_input": "2024-12-31T02:20:31.969686Z",
+     "iopub.status.busy": "2024-12-31T02:20:31.969511Z",
+     "iopub.status.idle": "2024-12-31T02:20:32.055139Z",
+     "shell.execute_reply": "2024-12-31T02:20:32.054549Z"
     }
    },
    "outputs": [
@@ -16191,9 +16191,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"8a472635-6d79-4701-854f-c62e62ebce54\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"8a472635-6d79-4701-854f-c62e62ebce54\")) {                    Plotly.newPlot(                        \"8a472635-6d79-4701-854f-c62e62ebce54\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.03906984627246857,0.0004489120910875499,0.03133600950241089,0.007519586011767387,0.034592170268297195,0.0003623008378781378,0.03451276570558548,0.19740214943885803,0.03844786807894707,0.04053793475031853,0.027628779411315918,0.02496313489973545],[0.18906497955322266,0.1721990704536438,0.06807752698659897,0.044945165514945984,0.0790855661034584,0.03096737712621689,0.028282105922698975,0.036443259567022324,0.026936713606119156,0.018826233223080635,0.04510088637471199,0.0065726605243980885],[0.15745531022548676,0.020724497735500336,0.48180025815963745,0.2991351783275604,0.1076490506529808,0.33004093170166016,0.09975510090589523,0.04926132783293724,0.2549382448196411,0.3606453239917755,0.12571798264980316,0.07931824773550034],[0.0058439951390028,0.1578730344772339,0.4189079701900482,0.30129143595695496,0.014345025643706322,0.03234415501356125,0.3312898576259613,0.5285982489585876,0.34241825342178345,0.10183659940958023,0.10516025871038437,0.22331231832504272],[0.10626537352800369,0.11930901557207108,0.02288069576025009,0.22826972603797913,0.02000402845442295,0.10010094195604324,0.17392224073410034,0.17407013475894928,0.025876855477690697,0.10249904543161392,0.009514803998172283,0.9921424388885498],[0.019766751676797867,0.005283255595713854,0.166484072804451,0.12087711691856384,0.1649997979402542,0.008032665587961674,0.4176996946334839,0.02582792565226555,0.04802580177783966,0.016231726855039597,0.031101685017347336,0.024261072278022766],[0.21729078888893127,0.03909975662827492,0.018048642203211784,0.059900619089603424,0.0329345278441906,0.0873454138636589,0.026895388960838318,0.09439495950937271,0.49926066398620605,0.00624012341722846,0.027026763185858727,0.1278562843799591],[0.25116443634033203,0.013308697380125523,0.006663368083536625,0.03743036091327667,0.02331540174782276,0.017407197505235672,0.02206745184957981,0.022141238674521446,0.04502354562282562,0.02084278129041195,0.008310752920806408,0.017167769372463226],[0.020890571177005768,0.016537854447960854,0.021583056077361107,0.015005743131041527,0.024211907759308815,0.1019899770617485,0.029100272804498672,0.22793740034103394,0.02781473658978939,0.01794101856648922,0.02482905425131321,0.03806224465370178],[0.02607565000653267,0.015407402068376541,0.020444145426154137,0.14558184146881104,0.01247023232281208,0.017151616513729095,0.013311783783137798,0.02445186860859394,0.018111703917384148,0.013193286024034023,0.03573964536190033,0.0187982190400362],[0.02147809788584709,0.018419133499264717,0.0181836299598217,0.021721404045820236,0.03156769275665283,0.03470622003078461,0.017550060525536537,0.011417530477046967,0.015795614570379257,0.04592300206422806,0.016215480864048004,0.030394626781344414],[0.033205095678567886,0.017571475356817245,0.015131049789488316,0.04148414358496666,0.015181177295744419,0.01758996769785881,0.01514849066734314,0.017676066607236862,0.06622707098722458,0.01845112442970276,0.01700744405388832,0.02974928542971611]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Previous Token Scores\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"c9a93815-e882-486d-9f08-66c767cc190d\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"c9a93815-e882-486d-9f08-66c767cc190d\")) {                    Plotly.newPlot(                        \"c9a93815-e882-486d-9f08-66c767cc190d\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.03906984627246857,0.0004489120910875499,0.03133600950241089,0.007519586011767387,0.034592170268297195,0.0003623008378781378,0.03451276570558548,0.19740214943885803,0.03844786807894707,0.04053793475031853,0.027628779411315918,0.02496313489973545],[0.18906497955322266,0.1721990704536438,0.06807752698659897,0.044945165514945984,0.0790855661034584,0.03096737712621689,0.028282105922698975,0.036443259567022324,0.026936713606119156,0.018826233223080635,0.04510088637471199,0.0065726605243980885],[0.15745531022548676,0.020724497735500336,0.48180025815963745,0.2991351783275604,0.1076490506529808,0.33004093170166016,0.09975510090589523,0.04926132783293724,0.2549382448196411,0.3606453239917755,0.12571798264980316,0.07931824773550034],[0.0058439951390028,0.1578730344772339,0.4189079701900482,0.30129143595695496,0.014345025643706322,0.03234415501356125,0.3312898576259613,0.5285982489585876,0.34241825342178345,0.10183659940958023,0.10516025871038437,0.22331231832504272],[0.10626537352800369,0.11930901557207108,0.02288069576025009,0.22826972603797913,0.02000402845442295,0.10010094195604324,0.17392224073410034,0.17407013475894928,0.025876855477690697,0.10249904543161392,0.009514803998172283,0.9921424388885498],[0.019766751676797867,0.005283255595713854,0.166484072804451,0.12087711691856384,0.1649997979402542,0.008032665587961674,0.4176996946334839,0.02582792565226555,0.04802580177783966,0.016231726855039597,0.031101685017347336,0.024261072278022766],[0.21729078888893127,0.03909975662827492,0.018048642203211784,0.059900619089603424,0.0329345278441906,0.0873454138636589,0.026895388960838318,0.09439495950937271,0.49926066398620605,0.00624012341722846,0.027026763185858727,0.1278562843799591],[0.25116443634033203,0.013308697380125523,0.006663368083536625,0.03743036091327667,0.02331540174782276,0.017407197505235672,0.02206745184957981,0.022141238674521446,0.04502354562282562,0.02084278129041195,0.008310752920806408,0.017167769372463226],[0.020890571177005768,0.016537854447960854,0.021583056077361107,0.015005743131041527,0.024211907759308815,0.1019899770617485,0.029100272804498672,0.22793740034103394,0.02781473658978939,0.01794101856648922,0.02482905425131321,0.03806224465370178],[0.02607565000653267,0.015407402068376541,0.020444145426154137,0.14558184146881104,0.01247023232281208,0.017151616513729095,0.013311783783137798,0.02445186860859394,0.018111703917384148,0.013193286024034023,0.03573964536190033,0.0187982190400362],[0.02147809788584709,0.018419133499264717,0.0181836299598217,0.021721404045820236,0.03156769275665283,0.03470622003078461,0.017550060525536537,0.011417530477046967,0.015795614570379257,0.04592300206422806,0.016215480864048004,0.030394626781344414],[0.033205095678567886,0.017571475356817245,0.015131049789488316,0.04148414358496666,0.015181177295744419,0.01758996769785881,0.01514849066734314,0.017676066607236862,0.06622707098722458,0.01845112442970276,0.01700744405388832,0.02974928542971611]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Previous Token Scores\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('8a472635-6d79-4701-854f-c62e62ebce54');\n",
+       "var gd = document.getElementById('c9a93815-e882-486d-9f08-66c767cc190d');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -17303,9 +17303,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"fae2e193-59a0-4811-84ef-98e7b34809aa\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"fae2e193-59a0-4811-84ef-98e7b34809aa\")) {                    Plotly.newPlot(                        \"fae2e193-59a0-4811-84ef-98e7b34809aa\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.003192325122654438,0.1323632299900055,0.005006915889680386,1.0427492270537186e-05,0.0013110269792377949,0.7034571170806885,0.004262051545083523,0.0001649669575272128,0.0024746304843574762,0.0008572910446673632,0.018891464918851852,0.008690940216183662],[0.0002916363882832229,0.00013781932648271322,0.0015036164550110698,0.005392488092184067,0.001858389819972217,0.009062976576387882,0.012414469383656979,0.00224055303260684,0.005135663319379091,0.005220625549554825,0.005546468310058117,0.029750702902674675],[0.0024816489312797785,0.009442185051739216,0.00034561342909000814,0.00025915156584233046,0.005211671814322472,0.0005709662800654769,0.0015209924895316362,0.006313098594546318,0.001560926903039217,0.00042158187716268003,0.0001535943301860243,0.005160389002412558],[0.6775667071342468,0.002840489149093628,0.0007841649930924177,0.004712563939392567,0.006322908215224743,0.00620671221986413,0.0005474375793710351,0.0003782894345931709,0.0020156530663371086,0.007952774874866009,0.003576836548745632,0.0026087891310453415],[0.008604029193520546,0.0070285736583173275,0.00759880430996418,0.0034427959471940994,0.016561321914196014,0.005979627370834351,0.0048697832971811295,0.0007624494028277695,0.006062139756977558,0.007536678109318018,0.012022883631289005,1.055450974014882e-12],[0.00950299110263586,0.008562120608985424,0.0041626025922596455,0.003008681582286954,0.006847443990409374,0.004358135629445314,0.007669272366911173,0.009584203362464905,0.007618824020028114,0.004328039940446615,0.04140293970704079,0.009761848486959934],[0.00445613032206893,0.008873283863067627,0.007405218668282032,0.006224926561117172,0.007319129537791014,0.005623883102089167,0.01734965480864048,0.005529423244297504,0.0029201731085777283,0.008636709302663803,0.006222232710570097,0.008358956314623356],[0.003699968568980694,0.041079357266426086,0.04148266464471817,0.009313643909990788,0.009097038768231869,0.008774377405643463,0.007298501208424568,0.023312222212553024,0.00884333811700344,0.00987985823303461,0.017598574981093407,0.006039811763912439],[0.008986336179077625,0.028667127713561058,0.00889119878411293,0.010114572942256927,0.009737403132021427,0.00761164166033268,0.009763195179402828,0.005155565217137337,0.009276354685425758,0.011895835399627686,0.010411957278847694,0.007498918566852808],[0.024409933015704155,0.011438388377428055,0.02003093995153904,0.005118591710925102,0.015081214718520641,0.012334463186562061,0.015452546067535877,0.008602438494563103,0.014702466316521168,0.020766181871294975,0.009192772209644318,0.005703327711671591],[0.017897022888064384,0.013280634768307209,0.0067551820538938046,0.012744802981615067,0.008020908571779728,0.007722198497503996,0.01734135113656521,0.0074547044932842255,0.007832478731870651,0.008252141997218132,0.013642732053995132,0.012807462364435196],[0.004923720378428698,0.007951012812554836,0.00794787798076868,0.004564068745821714,0.010363367386162281,0.009582011960446835,0.01028773095458746,0.008320694789290428,0.002570011653006077,0.012810957618057728,0.008063891902565956,0.0065582930110394955]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Duplicate Token Scores\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"8ebfd231-d4f6-431d-9f92-7bc6b6f70cd9\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"8ebfd231-d4f6-431d-9f92-7bc6b6f70cd9\")) {                    Plotly.newPlot(                        \"8ebfd231-d4f6-431d-9f92-7bc6b6f70cd9\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.003192325122654438,0.1323632299900055,0.005006915889680386,1.0427492270537186e-05,0.0013110269792377949,0.7034571170806885,0.004262051545083523,0.0001649669575272128,0.0024746304843574762,0.0008572910446673632,0.018891464918851852,0.008690940216183662],[0.0002916363882832229,0.00013781932648271322,0.0015036164550110698,0.005392488092184067,0.001858389819972217,0.009062976576387882,0.012414469383656979,0.00224055303260684,0.005135663319379091,0.005220625549554825,0.005546468310058117,0.029750702902674675],[0.0024816489312797785,0.009442185051739216,0.00034561342909000814,0.00025915156584233046,0.005211671814322472,0.0005709662800654769,0.0015209924895316362,0.006313098594546318,0.001560926903039217,0.00042158187716268003,0.0001535943301860243,0.005160389002412558],[0.6775667071342468,0.002840489149093628,0.0007841649930924177,0.004712563939392567,0.006322908215224743,0.00620671221986413,0.0005474375793710351,0.0003782894345931709,0.0020156530663371086,0.007952774874866009,0.003576836548745632,0.0026087891310453415],[0.008604029193520546,0.0070285736583173275,0.00759880430996418,0.0034427959471940994,0.016561321914196014,0.005979627370834351,0.0048697832971811295,0.0007624494028277695,0.006062139756977558,0.007536678109318018,0.012022883631289005,1.055450974014882e-12],[0.00950299110263586,0.008562120608985424,0.0041626025922596455,0.003008681582286954,0.006847443990409374,0.004358135629445314,0.007669272366911173,0.009584203362464905,0.007618824020028114,0.004328039940446615,0.04140293970704079,0.009761848486959934],[0.00445613032206893,0.008873283863067627,0.007405218668282032,0.006224926561117172,0.007319129537791014,0.005623883102089167,0.01734965480864048,0.005529423244297504,0.0029201731085777283,0.008636709302663803,0.006222232710570097,0.008358956314623356],[0.003699968568980694,0.041079357266426086,0.04148266464471817,0.009313643909990788,0.009097038768231869,0.008774377405643463,0.007298501208424568,0.023312222212553024,0.00884333811700344,0.00987985823303461,0.017598574981093407,0.006039811763912439],[0.008986336179077625,0.028667127713561058,0.00889119878411293,0.010114572942256927,0.009737403132021427,0.00761164166033268,0.009763195179402828,0.005155565217137337,0.009276354685425758,0.011895835399627686,0.010411957278847694,0.007498918566852808],[0.024409933015704155,0.011438388377428055,0.02003093995153904,0.005118591710925102,0.015081214718520641,0.012334463186562061,0.015452546067535877,0.008602438494563103,0.014702466316521168,0.020766181871294975,0.009192772209644318,0.005703327711671591],[0.017897022888064384,0.013280634768307209,0.0067551820538938046,0.012744802981615067,0.008020908571779728,0.007722198497503996,0.01734135113656521,0.0074547044932842255,0.007832478731870651,0.008252141997218132,0.013642732053995132,0.012807462364435196],[0.004923720378428698,0.007951012812554836,0.00794787798076868,0.004564068745821714,0.010363367386162281,0.009582011960446835,0.01028773095458746,0.008320694789290428,0.002570011653006077,0.012810957618057728,0.008063891902565956,0.0065582930110394955]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Duplicate Token Scores\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('fae2e193-59a0-4811-84ef-98e7b34809aa');\n",
+       "var gd = document.getElementById('8ebfd231-d4f6-431d-9f92-7bc6b6f70cd9');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -18415,9 +18415,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"ecf35438-ee23-4f6c-85d0-db50b0b55227\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"ecf35438-ee23-4f6c-85d0-db50b0b55227\")) {                    Plotly.newPlot(                        \"ecf35438-ee23-4f6c-85d0-db50b0b55227\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.004035576246678829,3.85936327802483e-05,0.003946060314774513,1.7428237697458826e-07,5.9896454331465065e-05,4.0836926928022876e-05,0.003501761006191373,0.00024610935361124575,0.003167978022247553,0.003010402200743556,0.0020935393404215574,0.008525436744093895],[0.0005264790961518884,0.0001567041763337329,0.0015079419827088714,0.005595334805548191,0.0018401179695501924,0.003887568600475788,0.005349150858819485,0.0046491725370287895,0.005880181211978197,0.007283915765583515,0.005552181042730808,0.00012677432096097618],[0.0022015569265931845,0.008784865029156208,0.0021590511314570904,0.0010448142420500517,0.005142332520335913,0.002251654164865613,0.0008376692421734333,0.0063524022698402405,0.00261817779392004,0.0010309149511158466,0.00015219332999549806,0.005351166240870953],[0.007752286270260811,0.003091607242822647,0.00136294006370008,0.004341903142631054,0.011233086697757244,0.006535585504025221,0.0009068045765161514,0.0006078635924495757,0.0028195830527693033,0.005254121031612158,0.004195701330900192,0.0025541584473103285],[0.007342719938606024,0.004788315389305353,0.007458813022822142,0.0033073171507567167,0.007871860638260841,0.004219716414809227,0.004172018263489008,0.0005154716782271862,0.008124986663460732,0.006826961878687143,0.008085506968200207,3.761463363005646e-11],[0.43377670645713806,0.9306100010871887,0.006382290739566088,0.003473056945949793,0.005501003935933113,0.9255975484848022,0.005381471943110228,0.007857300341129303,0.008637800812721252,0.015764445066452026,0.012188390828669071,0.008265750482678413],[0.0025073012802749872,0.008432051166892052,0.008623287081718445,0.007653359789401293,0.011058016680181026,0.005525414831936359,0.017205143347382545,0.004794336389750242,0.004097685217857361,0.9257786273956299,0.020375533029437065,0.006313955411314964],[0.005555589683353901,0.18942901492118835,0.8509916663169861,0.008273174054920673,0.008239682763814926,0.00864996574819088,0.028328388929367065,0.08996234834194183,0.0066174231469631195,0.009413909167051315,0.9037811756134033,0.03037119098007679],[0.0073545570485293865,0.3791305124759674,0.005602711346000433,0.025401653721928596,0.008504705503582954,0.00623104115948081,0.1189238503575325,0.0051146638579666615,0.013350976631045341,0.01576736569404602,0.025844166055321693,0.008429795503616333],[0.23989056050777435,0.14378714561462402,0.09330623596906662,0.005819808691740036,0.07744759321212769,0.01644795574247837,0.4442824423313141,0.011141379363834858,0.03618974611163139,0.47264590859413147,0.008039995096623898,0.030952973291277885],[0.36065515875816345,0.4820112884044647,0.02285100519657135,0.12641923129558563,0.04125585779547691,0.007237402722239494,0.287715345621109,0.3897298574447632,0.0300607867538929,0.006112930364906788,0.16554805636405945,0.2224510908126831],[0.007408502046018839,0.033737070858478546,0.020412690937519073,0.0027554105035960674,0.02518630214035511,0.07808848470449448,0.033082809299230576,0.046439945697784424,0.0032543179113417864,0.2744251787662506,0.3800223767757416,0.009483428671956062]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Induction Head Scores\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"2926cb84-a4e5-40c5-bca4-0dcd0eb9b20f\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"2926cb84-a4e5-40c5-bca4-0dcd0eb9b20f\")) {                    Plotly.newPlot(                        \"2926cb84-a4e5-40c5-bca4-0dcd0eb9b20f\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.004035576246678829,3.85936327802483e-05,0.003946060314774513,1.7428237697458826e-07,5.9896454331465065e-05,4.0836926928022876e-05,0.003501761006191373,0.00024610935361124575,0.003167978022247553,0.003010402200743556,0.0020935393404215574,0.008525436744093895],[0.0005264790961518884,0.0001567041763337329,0.0015079419827088714,0.005595334805548191,0.0018401179695501924,0.003887568600475788,0.005349150858819485,0.0046491725370287895,0.005880181211978197,0.007283915765583515,0.005552181042730808,0.00012677432096097618],[0.0022015569265931845,0.008784865029156208,0.0021590511314570904,0.0010448142420500517,0.005142332520335913,0.002251654164865613,0.0008376692421734333,0.0063524022698402405,0.00261817779392004,0.0010309149511158466,0.00015219332999549806,0.005351166240870953],[0.007752286270260811,0.003091607242822647,0.00136294006370008,0.004341903142631054,0.011233086697757244,0.006535585504025221,0.0009068045765161514,0.0006078635924495757,0.0028195830527693033,0.005254121031612158,0.004195701330900192,0.0025541584473103285],[0.007342719938606024,0.004788315389305353,0.007458813022822142,0.0033073171507567167,0.007871860638260841,0.004219716414809227,0.004172018263489008,0.0005154716782271862,0.008124986663460732,0.006826961878687143,0.008085506968200207,3.761463363005646e-11],[0.43377670645713806,0.9306100010871887,0.006382290739566088,0.003473056945949793,0.005501003935933113,0.9255975484848022,0.005381471943110228,0.007857300341129303,0.008637800812721252,0.015764445066452026,0.012188390828669071,0.008265750482678413],[0.0025073012802749872,0.008432051166892052,0.008623287081718445,0.007653359789401293,0.011058016680181026,0.005525414831936359,0.017205143347382545,0.004794336389750242,0.004097685217857361,0.9257786273956299,0.020375533029437065,0.006313955411314964],[0.005555589683353901,0.18942901492118835,0.8509916663169861,0.008273174054920673,0.008239682763814926,0.00864996574819088,0.028328388929367065,0.08996234834194183,0.0066174231469631195,0.009413909167051315,0.9037811756134033,0.03037119098007679],[0.0073545570485293865,0.3791305124759674,0.005602711346000433,0.025401653721928596,0.008504705503582954,0.00623104115948081,0.1189238503575325,0.0051146638579666615,0.013350976631045341,0.01576736569404602,0.025844166055321693,0.008429795503616333],[0.23989056050777435,0.14378714561462402,0.09330623596906662,0.005819808691740036,0.07744759321212769,0.01644795574247837,0.4442824423313141,0.011141379363834858,0.03618974611163139,0.47264590859413147,0.008039995096623898,0.030952973291277885],[0.36065515875816345,0.4820112884044647,0.02285100519657135,0.12641923129558563,0.04125585779547691,0.007237402722239494,0.287715345621109,0.3897298574447632,0.0300607867538929,0.006112930364906788,0.16554805636405945,0.2224510908126831],[0.007408502046018839,0.033737070858478546,0.020412690937519073,0.0027554105035960674,0.02518630214035511,0.07808848470449448,0.033082809299230576,0.046439945697784424,0.0032543179113417864,0.2744251787662506,0.3800223767757416,0.009483428671956062]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Induction Head Scores\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('ecf35438-ee23-4f6c-85d0-db50b0b55227');\n",
+       "var gd = document.getElementById('2926cb84-a4e5-40c5-bca4-0dcd0eb9b20f');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -18509,10 +18509,10 @@
    "execution_count": 38,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:14.681801Z",
-     "iopub.status.busy": "2024-12-14T01:01:14.681439Z",
-     "iopub.status.idle": "2024-12-14T01:01:14.974170Z",
-     "shell.execute_reply": "2024-12-14T01:01:14.973548Z"
+     "iopub.execute_input": "2024-12-31T02:20:32.057761Z",
+     "iopub.status.busy": "2024-12-31T02:20:32.057371Z",
+     "iopub.status.idle": "2024-12-31T02:20:32.342722Z",
+     "shell.execute_reply": "2024-12-31T02:20:32.342134Z"
     }
    },
    "outputs": [
@@ -18577,10 +18577,10 @@
    "execution_count": 39,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:14.976750Z",
-     "iopub.status.busy": "2024-12-14T01:01:14.976358Z",
-     "iopub.status.idle": "2024-12-14T01:01:15.444633Z",
-     "shell.execute_reply": "2024-12-14T01:01:15.444055Z"
+     "iopub.execute_input": "2024-12-31T02:20:32.345096Z",
+     "iopub.status.busy": "2024-12-31T02:20:32.344752Z",
+     "iopub.status.idle": "2024-12-31T02:20:32.767452Z",
+     "shell.execute_reply": "2024-12-31T02:20:32.766873Z"
     }
    },
    "outputs": [
@@ -19673,9 +19673,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"2ed18f74-fde1-4c81-ae6d-442fa345fd56\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"2ed18f74-fde1-4c81-ae6d-442fa345fd56\")) {                    Plotly.newPlot(                        \"2ed18f74-fde1-4c81-ae6d-442fa345fd56\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[-0.002156495349481702,-0.00046507216757163405,0.00024167195078916848,0.0002806607517413795,-0.0004162961558904499,-0.0004892406286671758,-0.0026209622155874968,-0.0029356717132031918,0.0004256087704561651,0.0005418135551735759,0.00023754549329169095,-7.4885378126055e-05],[-0.0006585237570106983,0.00040605897083878517,-0.0009330313769169152,0.0008937875390984118,-0.0009785089641809464,-0.0005338447517715394,-0.002798808738589287,-0.0042141154408454895,0.0025785760954022408,0.002450673608109355,0.0005351657164283097,0.0012349945027381182],[0.0009405009332112968,-0.0011168718338012695,-0.0011542299762368202,-0.001569736865349114,-0.0005699718603864312,0.0014514633221551776,0.0024399259127676487,0.003158304840326309,0.0009236858459189534,-0.0035781036131083965,-0.001065023592673242,-0.00035587642923928797],[-0.0005624899640679359,-1.1904543498530984e-05,0.0011530885240063071,0.0007360292365774512,0.0016493821749463677,0.0008800605428405106,-0.0006905477494001389,-0.0030319932848215103,0.0008080820553004742,0.00010364956688135862,-0.000580791849642992,-0.0011067147133871913],[-0.0026376410387456417,0.0002693022252060473,-0.0016417787410318851,-0.0034070289693772793,0.0017449733568355441,0.00046448662760667503,-0.0007899348856881261,0.0018328834557905793,-0.0008632306708022952,-0.00039788917638361454,0.0007879154873080552,-0.00012039285502396524],[0.0008688486414030194,0.0009473506361246109,-0.002280982444062829,-0.0011803142260760069,0.0002408194704912603,-0.0004318461287766695,-0.00037281878758221865,-0.0007385292556136847,0.0008114340016618371,-0.00040442857425659895,-0.007074241526424885,0.003946421667933464],[-0.014917591586709023,-0.0022801952436566353,0.0022679539397358894,-8.297240128740668e-05,-0.0049810330383479595,0.0027670827694237232,0.0062661729753017426,-0.0034861767198890448,-0.0013347702333703637,-0.001791873830370605,-0.0012231743894517422,0.0004053567536175251],[-0.0002461824333295226,-0.005790156312286854,-0.0004975919146090746,0.14218276739120483,-0.0014962118584662676,-0.01900637522339821,0.003133367281407118,-0.0018581977346912026,-0.011305071413516998,0.19225995242595673,-0.0011892454931512475,-0.0010282358853146434],[-0.0038004028610885143,-0.0008571104845032096,-0.013956702314317226,0.008289228193461895,0.004315000958740711,-0.009073707275092602,-0.08315176516771317,0.003456807229667902,-0.01805501990020275,0.00217801658436656,0.29780468344688416,0.024093760177493095],[0.08904657512903214,-0.0007931562722660601,0.07247605919837952,0.01501650083810091,-0.021209243685007095,0.05205458402633667,1.441115140914917,0.04743725806474686,-0.0322909839451313,0.0,0.0019993027672171593,-0.008072329685091972],[0.8600811958312988,0.32600724697113037,0.16344299912452698,0.07133562117815018,-0.004448350518941879,0.0006813746877014637,0.3661348223686218,-0.7105053663253784,-0.0020313761197030544,-0.03214346989989281,1.2294316291809082,0.0018451516516506672],[0.016877306625247,-0.0017300043255090714,-0.5010855793952942,0.02749773859977722,-0.005966317839920521,-0.004944117274135351,-0.08855222165584564,0.006622340530157089,0.04412432760000229,-0.02726682648062706,-1.1349124908447266,0.022879652678966522]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"margin\":{\"t\":60}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"2d8ff8a7-fc76-4e41-9668-0433e68e918b\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"2d8ff8a7-fc76-4e41-9668-0433e68e918b\")) {                    Plotly.newPlot(                        \"2d8ff8a7-fc76-4e41-9668-0433e68e918b\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[-0.002156495349481702,-0.00046507216757163405,0.00024167195078916848,0.0002806607517413795,-0.0004162961558904499,-0.0004892406286671758,-0.0026209622155874968,-0.0029356717132031918,0.0004256087704561651,0.0005418135551735759,0.00023754549329169095,-7.4885378126055e-05],[-0.0006585237570106983,0.00040605897083878517,-0.0009330313769169152,0.0008937875390984118,-0.0009785089641809464,-0.0005338447517715394,-0.002798808738589287,-0.0042141154408454895,0.0025785760954022408,0.002450673608109355,0.0005351657164283097,0.0012349945027381182],[0.0009405009332112968,-0.0011168718338012695,-0.0011542299762368202,-0.001569736865349114,-0.0005699718603864312,0.0014514633221551776,0.0024399259127676487,0.003158304840326309,0.0009236858459189534,-0.0035781036131083965,-0.001065023592673242,-0.00035587642923928797],[-0.0005624899640679359,-1.1904543498530984e-05,0.0011530885240063071,0.0007360292365774512,0.0016493821749463677,0.0008800605428405106,-0.0006905477494001389,-0.0030319932848215103,0.0008080820553004742,0.00010364956688135862,-0.000580791849642992,-0.0011067147133871913],[-0.0026376410387456417,0.0002693022252060473,-0.0016417787410318851,-0.0034070289693772793,0.0017449733568355441,0.00046448662760667503,-0.0007899348856881261,0.0018328834557905793,-0.0008632306708022952,-0.00039788917638361454,0.0007879154873080552,-0.00012039285502396524],[0.0008688486414030194,0.0009473506361246109,-0.002280982444062829,-0.0011803142260760069,0.0002408194704912603,-0.0004318461287766695,-0.00037281878758221865,-0.0007385292556136847,0.0008114340016618371,-0.00040442857425659895,-0.007074241526424885,0.003946421667933464],[-0.014917591586709023,-0.0022801952436566353,0.0022679539397358894,-8.297240128740668e-05,-0.0049810330383479595,0.0027670827694237232,0.0062661729753017426,-0.0034861767198890448,-0.0013347702333703637,-0.001791873830370605,-0.0012231743894517422,0.0004053567536175251],[-0.0002461824333295226,-0.005790156312286854,-0.0004975919146090746,0.14218276739120483,-0.0014962118584662676,-0.01900637522339821,0.003133367281407118,-0.0018581977346912026,-0.011305071413516998,0.19225995242595673,-0.0011892454931512475,-0.0010282358853146434],[-0.0038004028610885143,-0.0008571104845032096,-0.013956702314317226,0.008289228193461895,0.004315000958740711,-0.009073707275092602,-0.08315176516771317,0.003456807229667902,-0.01805501990020275,0.00217801658436656,0.29780468344688416,0.024093760177493095],[0.08904657512903214,-0.0007931562722660601,0.07247605919837952,0.01501650083810091,-0.021209243685007095,0.05205458402633667,1.441115140914917,0.04743725806474686,-0.0322909839451313,0.0,0.0019993027672171593,-0.008072329685091972],[0.8600811958312988,0.32600724697113037,0.16344299912452698,0.07133562117815018,-0.004448350518941879,0.0006813746877014637,0.3661348223686218,-0.7105053663253784,-0.0020313761197030544,-0.03214346989989281,1.2294316291809082,0.0018451516516506672],[0.016877306625247,-0.0017300043255090714,-0.5010855793952942,0.02749773859977722,-0.005966317839920521,-0.004944117274135351,-0.08855222165584564,0.006622340530157089,0.04412432760000229,-0.02726682648062706,-1.1349124908447266,0.022879652678966522]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"margin\":{\"t\":60}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('2ed18f74-fde1-4c81-ae6d-442fa345fd56');\n",
+       "var gd = document.getElementById('2d8ff8a7-fc76-4e41-9668-0433e68e918b');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -21018,9 +21018,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"69a381a6-98d5-41d7-b432-3b083c9cf66c\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"69a381a6-98d5-41d7-b432-3b083c9cf66c\")) {                    Plotly.newPlot(                        \"69a381a6-98d5-41d7-b432-3b083c9cf66c\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003eAblated=%{x}\\u003cbr\\u003eOriginal=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"L0H0\",\"L0H1\",\"L0H2\",\"L0H3\",\"L0H4\",\"L0H5\",\"L0H6\",\"L0H7\",\"L0H8\",\"L0H9\",\"L0H10\",\"L0H11\",\"L1H0\",\"L1H1\",\"L1H2\",\"L1H3\",\"L1H4\",\"L1H5\",\"L1H6\",\"L1H7\",\"L1H8\",\"L1H9\",\"L1H10\",\"L1H11\",\"L2H0\",\"L2H1\",\"L2H2\",\"L2H3\",\"L2H4\",\"L2H5\",\"L2H6\",\"L2H7\",\"L2H8\",\"L2H9\",\"L2H10\",\"L2H11\",\"L3H0\",\"L3H1\",\"L3H2\",\"L3H3\",\"L3H4\",\"L3H5\",\"L3H6\",\"L3H7\",\"L3H8\",\"L3H9\",\"L3H10\",\"L3H11\",\"L4H0\",\"L4H1\",\"L4H2\",\"L4H3\",\"L4H4\",\"L4H5\",\"L4H6\",\"L4H7\",\"L4H8\",\"L4H9\",\"L4H10\",\"L4H11\",\"L5H0\",\"L5H1\",\"L5H2\",\"L5H3\",\"L5H4\",\"L5H5\",\"L5H6\",\"L5H7\",\"L5H8\",\"L5H9\",\"L5H10\",\"L5H11\",\"L6H0\",\"L6H1\",\"L6H2\",\"L6H3\",\"L6H4\",\"L6H5\",\"L6H6\",\"L6H7\",\"L6H8\",\"L6H9\",\"L6H10\",\"L6H11\",\"L7H0\",\"L7H1\",\"L7H2\",\"L7H3\",\"L7H4\",\"L7H5\",\"L7H6\",\"L7H7\",\"L7H8\",\"L7H9\",\"L7H10\",\"L7H11\",\"L8H0\",\"L8H1\",\"L8H2\",\"L8H3\",\"L8H4\",\"L8H5\",\"L8H6\",\"L8H7\",\"L8H8\",\"L8H9\",\"L8H10\",\"L8H11\",\"L9H0\",\"L9H1\",\"L9H2\",\"L9H3\",\"L9H4\",\"L9H5\",\"L9H6\",\"L9H7\",\"L9H8\",\"L9H9\",\"L9H10\",\"L9H11\",\"L10H0\",\"L10H1\",\"L10H2\",\"L10H3\",\"L10H4\",\"L10H5\",\"L10H6\",\"L10H7\",\"L10H8\",\"L10H9\",\"L10H10\",\"L10H11\",\"L11H0\",\"L11H1\",\"L11H2\",\"L11H3\",\"L11H4\",\"L11H5\",\"L11H6\",\"L11H7\",\"L11H8\",\"L11H9\",\"L11H10\",\"L11H11\"],\"legendgroup\":\"\",\"marker\":{\"color\":\"#636efa\",\"symbol\":\"circle\"},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[-0.002156495349481702,-0.00046507216757163405,0.00024167195078916848,0.0002806607517413795,-0.0004162961558904499,-0.0004892406286671758,-0.0026209622155874968,-0.0029356717132031918,0.0004256087704561651,0.0005418135551735759,0.00023754549329169095,-7.4885378126055e-05,-0.0006585237570106983,0.00040605897083878517,-0.0009330313769169152,0.0008937875390984118,-0.0009785089641809464,-0.0005338447517715394,-0.002798808738589287,-0.0042141154408454895,0.0025785760954022408,0.002450673608109355,0.0005351657164283097,0.0012349945027381182,0.0009405009332112968,-0.0011168718338012695,-0.0011542299762368202,-0.001569736865349114,-0.0005699718603864312,0.0014514633221551776,0.0024399259127676487,0.003158304840326309,0.0009236858459189534,-0.0035781036131083965,-0.001065023592673242,-0.00035587642923928797,-0.0005624899640679359,-1.1904543498530984e-05,0.0011530885240063071,0.0007360292365774512,0.0016493821749463677,0.0008800605428405106,-0.0006905477494001389,-0.0030319932848215103,0.0008080820553004742,0.00010364956688135862,-0.000580791849642992,-0.0011067147133871913,-0.0026376410387456417,0.0002693022252060473,-0.0016417787410318851,-0.0034070289693772793,0.0017449733568355441,0.00046448662760667503,-0.0007899348856881261,0.0018328834557905793,-0.0008632306708022952,-0.00039788917638361454,0.0007879154873080552,-0.00012039285502396524,0.0008688486414030194,0.0009473506361246109,-0.002280982444062829,-0.0011803142260760069,0.0002408194704912603,-0.0004318461287766695,-0.00037281878758221865,-0.0007385292556136847,0.0008114340016618371,-0.00040442857425659895,-0.007074241526424885,0.003946421667933464,-0.014917591586709023,-0.0022801952436566353,0.0022679539397358894,-8.297240128740668e-05,-0.0049810330383479595,0.0027670827694237232,0.0062661729753017426,-0.0034861767198890448,-0.0013347702333703637,-0.001791873830370605,-0.0012231743894517422,0.0004053567536175251,-0.0002461824333295226,-0.005790156312286854,-0.0004975919146090746,0.14218276739120483,-0.0014962118584662676,-0.01900637522339821,0.003133367281407118,-0.0018581977346912026,-0.011305071413516998,0.19225995242595673,-0.0011892454931512475,-0.0010282358853146434,-0.0038004028610885143,-0.0008571104845032096,-0.013956702314317226,0.008289228193461895,0.004315000958740711,-0.009073707275092602,-0.08315176516771317,0.003456807229667902,-0.01805501990020275,0.00217801658436656,0.29780468344688416,0.024093760177493095,0.08904657512903214,-0.0007931562722660601,0.07247605919837952,0.01501650083810091,-0.021209243685007095,0.05205458402633667,1.441115140914917,0.04743725806474686,-0.0322909839451313,0.0,0.0019993027672171593,-0.008072329685091972,0.8600811958312988,0.32600724697113037,0.16344299912452698,0.07133562117815018,-0.004448350518941879,0.0006813746877014637,0.3661348223686218,-0.7105053663253784,-0.0020313761197030544,-0.03214346989989281,1.2294316291809082,0.0018451516516506672,0.016877306625247,-0.0017300043255090714,-0.5010855793952942,0.02749773859977722,-0.005966317839920521,-0.004944117274135351,-0.08855222165584564,0.006622340530157089,0.04412432760000229,-0.02726682648062706,-1.1349124908447266,0.022879652678966522],\"xaxis\":\"x\",\"y\":[-0.002056329045444727,-0.0005102002760395408,0.0004685759777203202,0.00012511858949437737,-0.0006028746138326824,-0.00024295759794767946,-0.0023188991472125053,-0.0027583539485931396,0.000564592657610774,0.0009697366622276604,-0.0002504411095287651,4.7396752052009106e-06,-0.0010071106953546405,0.0003947066143155098,-0.001548723317682743,0.0014034844934940338,-0.0012652697041630745,-0.0011358254123479128,-0.0028159399516880512,-0.0029645331669598818,0.0029190238565206528,0.0025743518490344286,0.00036237656604498625,0.0017548904288560152,0.0005569161148741841,-0.0011266364017501473,-0.0017354193842038512,-0.0014514740323647857,-0.0002874041674658656,0.0017210585065186024,0.0026659027207642794,0.0031146786641329527,0.0005667298682965338,-0.003666497301310301,-0.0018847138853743672,7.028633262962103e-06,-0.0007264401647262275,0.0001136981591116637,0.0014301439514383674,0.0007490518037229776,0.002018478699028492,0.0007436765008606017,-0.0004617785452865064,-0.0039057875983417034,0.0011407355777919292,-4.027353134006262e-05,-0.0013293151278048754,-0.0017636835109442472,-0.0028281747363507748,0.0003364539588801563,-0.0014249025844037533,-0.003777292789891362,0.001599886454641819,0.00029889593133702874,-0.0008046383736655116,0.0020388178527355194,-0.001559373107738793,-0.0006437154370360076,0.001116809668019414,-0.0003500327584333718,0.0011338141048327088,0.0011259093880653381,-0.0025163700338453054,-0.0014790259301662445,0.00038791849510744214,-6.407558976206928e-05,-0.0005096771637909114,-0.0008841846138238907,0.0006399258272722363,-0.0010097046615555882,-0.006759032607078552,0.0033667325042188168,-0.015147397294640541,-0.0021350914612412453,0.0025931934360414743,-0.00042673421557992697,-0.005559004843235016,0.002665933221578598,0.0064108846709132195,-0.0038270088844001293,-0.0003842375008389354,-0.0016430213581770658,-0.001334429020062089,-9.183748625218868e-05,-9.488424984738231e-05,-0.005788922775536776,-0.0006383699947036803,0.134933739900589,-0.0017687629442662,-0.018917974084615707,0.0038733729161322117,-0.0021450738422572613,-0.010327240452170372,0.1832587718963623,-0.0007747872150503099,-0.0010452070273458958,-0.0038339472375810146,-0.0008046274306252599,-0.01267334446310997,0.008045826107263565,0.003604009747505188,-0.009398158639669418,-0.08272106945514679,0.0035550370812416077,-0.01840413361787796,0.001758674974553287,0.28961312770843506,0.022854033857584,0.08595199882984161,-0.000693259877152741,0.06816964596509933,0.01311142835766077,-0.02109798789024353,0.05112443491816521,1.384489893913269,0.045837316662073135,-0.038303449749946594,2.9854443073272705,0.001966138370335102,-0.008030208759009838,0.5608724355697632,0.17083144187927246,-0.033618729561567307,0.05821548402309418,-0.0024530175141990185,0.0018771894974634051,0.2882729470729828,-1.8986413478851318,-0.0015286938287317753,-0.035130057483911514,0.48021769523620605,-0.0009116912842728198,0.016075868159532547,-0.03986111283302307,-0.3879111707210541,0.01112320739775896,-0.0054778107441961765,-0.002512941136956215,-0.08056112378835678,0.007518645375967026,0.04301108419895172,-0.040082402527332306,-0.9702335000038147,0.011862380430102348],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Ablated\"},\"range\":[-3,3]},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Original\"},\"range\":[-3,3]},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Original vs Post-Ablation Direct Logit Attribution of Heads\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"df924299-a4ee-4575-87e6-763165906024\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"df924299-a4ee-4575-87e6-763165906024\")) {                    Plotly.newPlot(                        \"df924299-a4ee-4575-87e6-763165906024\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003eAblated=%{x}\\u003cbr\\u003eOriginal=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"L0H0\",\"L0H1\",\"L0H2\",\"L0H3\",\"L0H4\",\"L0H5\",\"L0H6\",\"L0H7\",\"L0H8\",\"L0H9\",\"L0H10\",\"L0H11\",\"L1H0\",\"L1H1\",\"L1H2\",\"L1H3\",\"L1H4\",\"L1H5\",\"L1H6\",\"L1H7\",\"L1H8\",\"L1H9\",\"L1H10\",\"L1H11\",\"L2H0\",\"L2H1\",\"L2H2\",\"L2H3\",\"L2H4\",\"L2H5\",\"L2H6\",\"L2H7\",\"L2H8\",\"L2H9\",\"L2H10\",\"L2H11\",\"L3H0\",\"L3H1\",\"L3H2\",\"L3H3\",\"L3H4\",\"L3H5\",\"L3H6\",\"L3H7\",\"L3H8\",\"L3H9\",\"L3H10\",\"L3H11\",\"L4H0\",\"L4H1\",\"L4H2\",\"L4H3\",\"L4H4\",\"L4H5\",\"L4H6\",\"L4H7\",\"L4H8\",\"L4H9\",\"L4H10\",\"L4H11\",\"L5H0\",\"L5H1\",\"L5H2\",\"L5H3\",\"L5H4\",\"L5H5\",\"L5H6\",\"L5H7\",\"L5H8\",\"L5H9\",\"L5H10\",\"L5H11\",\"L6H0\",\"L6H1\",\"L6H2\",\"L6H3\",\"L6H4\",\"L6H5\",\"L6H6\",\"L6H7\",\"L6H8\",\"L6H9\",\"L6H10\",\"L6H11\",\"L7H0\",\"L7H1\",\"L7H2\",\"L7H3\",\"L7H4\",\"L7H5\",\"L7H6\",\"L7H7\",\"L7H8\",\"L7H9\",\"L7H10\",\"L7H11\",\"L8H0\",\"L8H1\",\"L8H2\",\"L8H3\",\"L8H4\",\"L8H5\",\"L8H6\",\"L8H7\",\"L8H8\",\"L8H9\",\"L8H10\",\"L8H11\",\"L9H0\",\"L9H1\",\"L9H2\",\"L9H3\",\"L9H4\",\"L9H5\",\"L9H6\",\"L9H7\",\"L9H8\",\"L9H9\",\"L9H10\",\"L9H11\",\"L10H0\",\"L10H1\",\"L10H2\",\"L10H3\",\"L10H4\",\"L10H5\",\"L10H6\",\"L10H7\",\"L10H8\",\"L10H9\",\"L10H10\",\"L10H11\",\"L11H0\",\"L11H1\",\"L11H2\",\"L11H3\",\"L11H4\",\"L11H5\",\"L11H6\",\"L11H7\",\"L11H8\",\"L11H9\",\"L11H10\",\"L11H11\"],\"legendgroup\":\"\",\"marker\":{\"color\":\"#636efa\",\"symbol\":\"circle\"},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[-0.002156495349481702,-0.00046507216757163405,0.00024167195078916848,0.0002806607517413795,-0.0004162961558904499,-0.0004892406286671758,-0.0026209622155874968,-0.0029356717132031918,0.0004256087704561651,0.0005418135551735759,0.00023754549329169095,-7.4885378126055e-05,-0.0006585237570106983,0.00040605897083878517,-0.0009330313769169152,0.0008937875390984118,-0.0009785089641809464,-0.0005338447517715394,-0.002798808738589287,-0.0042141154408454895,0.0025785760954022408,0.002450673608109355,0.0005351657164283097,0.0012349945027381182,0.0009405009332112968,-0.0011168718338012695,-0.0011542299762368202,-0.001569736865349114,-0.0005699718603864312,0.0014514633221551776,0.0024399259127676487,0.003158304840326309,0.0009236858459189534,-0.0035781036131083965,-0.001065023592673242,-0.00035587642923928797,-0.0005624899640679359,-1.1904543498530984e-05,0.0011530885240063071,0.0007360292365774512,0.0016493821749463677,0.0008800605428405106,-0.0006905477494001389,-0.0030319932848215103,0.0008080820553004742,0.00010364956688135862,-0.000580791849642992,-0.0011067147133871913,-0.0026376410387456417,0.0002693022252060473,-0.0016417787410318851,-0.0034070289693772793,0.0017449733568355441,0.00046448662760667503,-0.0007899348856881261,0.0018328834557905793,-0.0008632306708022952,-0.00039788917638361454,0.0007879154873080552,-0.00012039285502396524,0.0008688486414030194,0.0009473506361246109,-0.002280982444062829,-0.0011803142260760069,0.0002408194704912603,-0.0004318461287766695,-0.00037281878758221865,-0.0007385292556136847,0.0008114340016618371,-0.00040442857425659895,-0.007074241526424885,0.003946421667933464,-0.014917591586709023,-0.0022801952436566353,0.0022679539397358894,-8.297240128740668e-05,-0.0049810330383479595,0.0027670827694237232,0.0062661729753017426,-0.0034861767198890448,-0.0013347702333703637,-0.001791873830370605,-0.0012231743894517422,0.0004053567536175251,-0.0002461824333295226,-0.005790156312286854,-0.0004975919146090746,0.14218276739120483,-0.0014962118584662676,-0.01900637522339821,0.003133367281407118,-0.0018581977346912026,-0.011305071413516998,0.19225995242595673,-0.0011892454931512475,-0.0010282358853146434,-0.0038004028610885143,-0.0008571104845032096,-0.013956702314317226,0.008289228193461895,0.004315000958740711,-0.009073707275092602,-0.08315176516771317,0.003456807229667902,-0.01805501990020275,0.00217801658436656,0.29780468344688416,0.024093760177493095,0.08904657512903214,-0.0007931562722660601,0.07247605919837952,0.01501650083810091,-0.021209243685007095,0.05205458402633667,1.441115140914917,0.04743725806474686,-0.0322909839451313,0.0,0.0019993027672171593,-0.008072329685091972,0.8600811958312988,0.32600724697113037,0.16344299912452698,0.07133562117815018,-0.004448350518941879,0.0006813746877014637,0.3661348223686218,-0.7105053663253784,-0.0020313761197030544,-0.03214346989989281,1.2294316291809082,0.0018451516516506672,0.016877306625247,-0.0017300043255090714,-0.5010855793952942,0.02749773859977722,-0.005966317839920521,-0.004944117274135351,-0.08855222165584564,0.006622340530157089,0.04412432760000229,-0.02726682648062706,-1.1349124908447266,0.022879652678966522],\"xaxis\":\"x\",\"y\":[-0.002056329045444727,-0.0005102002760395408,0.0004685759777203202,0.00012511858949437737,-0.0006028746138326824,-0.00024295759794767946,-0.0023188991472125053,-0.0027583539485931396,0.000564592657610774,0.0009697366622276604,-0.0002504411095287651,4.7396752052009106e-06,-0.0010071106953546405,0.0003947066143155098,-0.001548723317682743,0.0014034844934940338,-0.0012652697041630745,-0.0011358254123479128,-0.0028159399516880512,-0.0029645331669598818,0.0029190238565206528,0.0025743518490344286,0.00036237656604498625,0.0017548904288560152,0.0005569161148741841,-0.0011266364017501473,-0.0017354193842038512,-0.0014514740323647857,-0.0002874041674658656,0.0017210585065186024,0.0026659027207642794,0.0031146786641329527,0.0005667298682965338,-0.003666497301310301,-0.0018847138853743672,7.028633262962103e-06,-0.0007264401647262275,0.0001136981591116637,0.0014301439514383674,0.0007490518037229776,0.002018478699028492,0.0007436765008606017,-0.0004617785452865064,-0.0039057875983417034,0.0011407355777919292,-4.027353134006262e-05,-0.0013293151278048754,-0.0017636835109442472,-0.0028281747363507748,0.0003364539588801563,-0.0014249025844037533,-0.003777292789891362,0.001599886454641819,0.00029889593133702874,-0.0008046383736655116,0.0020388178527355194,-0.001559373107738793,-0.0006437154370360076,0.001116809668019414,-0.0003500327584333718,0.0011338141048327088,0.0011259093880653381,-0.0025163700338453054,-0.0014790259301662445,0.00038791849510744214,-6.407558976206928e-05,-0.0005096771637909114,-0.0008841846138238907,0.0006399258272722363,-0.0010097046615555882,-0.006759032607078552,0.0033667325042188168,-0.015147397294640541,-0.0021350914612412453,0.0025931934360414743,-0.00042673421557992697,-0.005559004843235016,0.002665933221578598,0.0064108846709132195,-0.0038270088844001293,-0.0003842375008389354,-0.0016430213581770658,-0.001334429020062089,-9.183748625218868e-05,-9.488424984738231e-05,-0.005788922775536776,-0.0006383699947036803,0.134933739900589,-0.0017687629442662,-0.018917974084615707,0.0038733729161322117,-0.0021450738422572613,-0.010327240452170372,0.1832587718963623,-0.0007747872150503099,-0.0010452070273458958,-0.0038339472375810146,-0.0008046274306252599,-0.01267334446310997,0.008045826107263565,0.003604009747505188,-0.009398158639669418,-0.08272106945514679,0.0035550370812416077,-0.01840413361787796,0.001758674974553287,0.28961312770843506,0.022854033857584,0.08595199882984161,-0.000693259877152741,0.06816964596509933,0.01311142835766077,-0.02109798789024353,0.05112443491816521,1.384489893913269,0.045837316662073135,-0.038303449749946594,2.9854443073272705,0.001966138370335102,-0.008030208759009838,0.5608724355697632,0.17083144187927246,-0.033618729561567307,0.05821548402309418,-0.0024530175141990185,0.0018771894974634051,0.2882729470729828,-1.8986413478851318,-0.0015286938287317753,-0.035130057483911514,0.48021769523620605,-0.0009116912842728198,0.016075868159532547,-0.03986111283302307,-0.3879111707210541,0.01112320739775896,-0.0054778107441961765,-0.002512941136956215,-0.08056112378835678,0.007518645375967026,0.04301108419895172,-0.040082402527332306,-0.9702335000038147,0.011862380430102348],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Ablated\"},\"range\":[-3,3]},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Original\"},\"range\":[-3,3]},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Original vs Post-Ablation Direct Logit Attribution of Heads\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('69a381a6-98d5-41d7-b432-3b083c9cf66c');\n",
+       "var gd = document.getElementById('df924299-a4ee-4575-87e6-763165906024');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -21085,10 +21085,10 @@
    "execution_count": 40,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:15.447724Z",
-     "iopub.status.busy": "2024-12-14T01:01:15.447522Z",
-     "iopub.status.idle": "2024-12-14T01:01:15.452855Z",
-     "shell.execute_reply": "2024-12-14T01:01:15.452388Z"
+     "iopub.execute_input": "2024-12-31T02:20:32.769599Z",
+     "iopub.status.busy": "2024-12-31T02:20:32.769282Z",
+     "iopub.status.idle": "2024-12-31T02:20:32.774466Z",
+     "shell.execute_reply": "2024-12-31T02:20:32.773927Z"
     }
    },
    "outputs": [
@@ -21174,93 +21174,7 @@
   "widgets": {
    "application/vnd.jupyter.widget-state+json": {
     "state": {
-     "0947963163124b84ad7bdbd0d14a5852": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
-      }
-     },
-     "0b727299d6b94848ba14d79205e0d847": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HBoxModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HBoxModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HBoxView",
-       "box_style": "",
-       "children": [
-        "IPY_MODEL_d3bf56862e814e46a08ce4949ce5df67",
-        "IPY_MODEL_9a499b42e58849dcbfc030ebb0630a62",
-        "IPY_MODEL_b4faa038d86840faa63b4ff7f8ac404c"
-       ],
-       "layout": "IPY_MODEL_5fe7263a54b34f2b918ca2c57d194d74",
-       "tabbable": null,
-       "tooltip": null
-      }
-     },
-     "0ccd5c3ba66d4687bda0b8a38511eecd": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_b60c8dc4072a47f69bc60e5115ab87c6",
-       "placeholder": "​",
-       "style": "IPY_MODEL_3dd3706dad6149a2ae71bfb8a4a5256a",
-       "tabbable": null,
-       "tooltip": null,
-       "value": "model.safetensors: 100%"
-      }
-     },
-     "0dc2fdfccac3440ea6a358b4b727da08": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_910235bb167a412ab2a54c930635b3bf",
-       "placeholder": "​",
-       "style": "IPY_MODEL_f988b1609264444ea2e348b491a028a4",
-       "tabbable": null,
-       "tooltip": null,
-       "value": " 1.36M/1.36M [00:00&lt;00:00, 47.4MB/s]"
-      }
-     },
-     "1acc38b256394079bf479fc9c3f6bbff": {
+     "00869a4f30534799a76cd92f9815d895": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -21313,7 +21227,7 @@
        "width": null
       }
      },
-     "1dffcf97acf041dfa4829930fc637e56": {
+     "058fdb1f616145a989119197863c8fa2": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -21366,57 +21280,39 @@
        "width": null
       }
      },
-     "2566b58370ab4ee1a23ec40b298e524d": {
+     "063d9fb16c374bbcbdee214535e4af62": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HBoxModel",
+      "model_name": "ProgressStyleModel",
       "state": {
-       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HBoxModel",
+       "_model_name": "ProgressStyleModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
+       "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
-       "_view_name": "HBoxView",
-       "box_style": "",
-       "children": [
-        "IPY_MODEL_0ccd5c3ba66d4687bda0b8a38511eecd",
-        "IPY_MODEL_2a992a646fbc405db6ec21ba17538e0e",
-        "IPY_MODEL_ddeeebc813c54775ab8841a88e512534"
-       ],
-       "layout": "IPY_MODEL_1acc38b256394079bf479fc9c3f6bbff",
-       "tabbable": null,
-       "tooltip": null
+       "_view_name": "StyleView",
+       "bar_color": null,
+       "description_width": ""
       }
      },
-     "2a992a646fbc405db6ec21ba17538e0e": {
+     "0a2dc7a44b50480498bd6e0a8258b228": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "FloatProgressModel",
+      "model_name": "ProgressStyleModel",
       "state": {
-       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "FloatProgressModel",
+       "_model_name": "ProgressStyleModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
+       "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
-       "_view_name": "ProgressView",
-       "bar_style": "success",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_fe14fba431ee44719e467a94d38602c1",
-       "max": 548105171.0,
-       "min": 0.0,
-       "orientation": "horizontal",
-       "style": "IPY_MODEL_71480013d795482ea7827866fd7d1fb0",
-       "tabbable": null,
-       "tooltip": null,
-       "value": 548105171.0
+       "_view_name": "StyleView",
+       "bar_color": null,
+       "description_width": ""
       }
      },
-     "3250ac219bb14a0d87380c723d7187c5": {
+     "0f16d4bd6ea64fa6b925ab2efd1ca15e": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -21469,7 +21365,49 @@
        "width": null
       }
      },
-     "3dd3706dad6149a2ae71bfb8a4a5256a": {
+     "1257c8ebaf854ceba5d1ffbad2bc4d64": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "12f2342a4d7e48c9aaec6b2665da59ed": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HBoxModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HBoxModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HBoxView",
+       "box_style": "",
+       "children": [
+        "IPY_MODEL_2536618b59144bf68c23a2f9688d5ddb",
+        "IPY_MODEL_239292d605c54be48e65ecf57c0cb3cf",
+        "IPY_MODEL_28ddfba82ab54559ba0701ee4fa3195b"
+       ],
+       "layout": "IPY_MODEL_5e99fedf15684824abf596d6692378e1",
+       "tabbable": null,
+       "tooltip": null
+      }
+     },
+     "175ab33428e24054a9588b575b136309": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLStyleModel",
@@ -21487,7 +21425,7 @@
        "text_color": null
       }
      },
-     "4064ed6431a14c8d940329269c2399e3": {
+     "19befccdb76441798bcd9c34f9a3970e": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLStyleModel",
@@ -21505,7 +21443,7 @@
        "text_color": null
       }
      },
-     "4647f9260ab248d7a350e7a07c999a9b": {
+     "1fd6a946b74140c8b91d0cfcc9c8a048": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -21558,23 +21496,143 @@
        "width": null
       }
      },
-     "47513102e87a47069219c229c4725eea": {
+     "220d51ccdafb40df864859c57034a447": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
+      "model_name": "HTMLStyleModel",
       "state": {
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
+       "_model_name": "HTMLStyleModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
        "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "239292d605c54be48e65ecf57c0cb3cf": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "FloatProgressModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "FloatProgressModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "ProgressView",
+       "bar_style": "success",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_389fb2476c1e46eca84fb5e7bb0eec35",
+       "max": 1355256.0,
+       "min": 0.0,
+       "orientation": "horizontal",
+       "style": "IPY_MODEL_708a6bc613444a518f2b42ae989a958a",
+       "tabbable": null,
+       "tooltip": null,
+       "value": 1355256.0
+      }
+     },
+     "2536618b59144bf68c23a2f9688d5ddb": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_da86d7929cf44490b4235842717812d2",
+       "placeholder": "​",
+       "style": "IPY_MODEL_46e2ce762f2046b0acf9a0538b70d087",
+       "tabbable": null,
+       "tooltip": null,
+       "value": "tokenizer.json: 100%"
+      }
+     },
+     "28ddfba82ab54559ba0701ee4fa3195b": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_6bbb6e62a049423bb1bb4a903cd0f7ef",
+       "placeholder": "​",
+       "style": "IPY_MODEL_4cc4e90000854566806a0f904f8413a5",
+       "tabbable": null,
+       "tooltip": null,
+       "value": " 1.36M/1.36M [00:00&lt;00:00, 5.12MB/s]"
+      }
+     },
+     "2bf41dd86aaf412b8d50d2cf2ab37cad": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_baa17a1c01be427c8dd2fe2e9a46b3fc",
+       "placeholder": "​",
+       "style": "IPY_MODEL_1257c8ebaf854ceba5d1ffbad2bc4d64",
+       "tabbable": null,
+       "tooltip": null,
+       "value": "tokenizer_config.json: 100%"
+      }
+     },
+     "2dae86562a8d47b5a5c7935ebcdfea75": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_f94c5abf57df4cdba73ed4b8c8bafb54",
+       "placeholder": "​",
+       "style": "IPY_MODEL_35b095005ad54340aeff4c74af9f6f76",
+       "tabbable": null,
+       "tooltip": null,
+       "value": " 26.0/26.0 [00:00&lt;00:00, 4.68kB/s]"
       }
      },
-     "481fa4fd501940cebdf3f44be6c9e882": {
+     "32a2e286dd0e4b47a7ed286e57e25a02": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -21627,7 +21685,25 @@
        "width": null
       }
      },
-     "488e5f543a8f48a79b17dd90afc28284": {
+     "35b095005ad54340aeff4c74af9f6f76": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "389fb2476c1e46eca84fb5e7bb0eec35": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -21680,7 +21756,7 @@
        "width": null
       }
      },
-     "4973dab052f14376aa999fd6bc13197b": {
+     "41f933e12f1d4d79a42601d7b7c9da82": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -21733,7 +21809,23 @@
        "width": null
       }
      },
-     "4a23001ae4824747ac51dbe487a0a126": {
+     "449107685d8c439e8aaf9931f3acf205": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "ProgressStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "ProgressStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "bar_color": null,
+       "description_width": ""
+      }
+     },
+     "44cc2a714dab4fbd92b45ec21a01bcef": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -21786,31 +21878,7 @@
        "width": null
       }
      },
-     "4c4a1ec2b7514c1297506faae67c1374": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HBoxModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HBoxModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HBoxView",
-       "box_style": "",
-       "children": [
-        "IPY_MODEL_53bd3bcec1c747a0a2f91fd51bd8fba0",
-        "IPY_MODEL_53f61b927c1346d88f4d1f2962f875ba",
-        "IPY_MODEL_c43b6d1b7dc04e8ea921c5940f890126"
-       ],
-       "layout": "IPY_MODEL_b1a0d471042a4a018d4de3bf29a3ecd2",
-       "tabbable": null,
-       "tooltip": null
-      }
-     },
-     "4faadec4ac0f4a4ab32961fb3b74e529": {
+     "46075fcab25949e4accbc84c39c7b694": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -21863,7 +21931,25 @@
        "width": null
       }
      },
-     "53bd3bcec1c747a0a2f91fd51bd8fba0": {
+     "46e2ce762f2046b0acf9a0538b70d087": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "48e6660d282a4e2db01422526059d280": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLModel",
@@ -21878,41 +21964,33 @@
        "_view_name": "HTMLView",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_481fa4fd501940cebdf3f44be6c9e882",
+       "layout": "IPY_MODEL_be0da02273224522ab03fcdf121a9d42",
        "placeholder": "​",
-       "style": "IPY_MODEL_ed04372d5ce8433d848d96e975723537",
+       "style": "IPY_MODEL_a3a152183bb84d98b4545d9f13d5980b",
        "tabbable": null,
        "tooltip": null,
-       "value": "generation_config.json: 100%"
+       "value": "model.safetensors: 100%"
       }
      },
-     "53f61b927c1346d88f4d1f2962f875ba": {
+     "4cc4e90000854566806a0f904f8413a5": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "FloatProgressModel",
+      "model_name": "HTMLStyleModel",
       "state": {
-       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "FloatProgressModel",
+       "_model_name": "HTMLStyleModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
+       "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
-       "_view_name": "ProgressView",
-       "bar_style": "success",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_f06d8ed2f2fc40c78e6f71a980042fa2",
-       "max": 124.0,
-       "min": 0.0,
-       "orientation": "horizontal",
-       "style": "IPY_MODEL_0947963163124b84ad7bdbd0d14a5852",
-       "tabbable": null,
-       "tooltip": null,
-       "value": 124.0
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
       }
      },
-     "54e358754fd241a7beb89dbd18d54469": {
+     "5af480c7ada14eb98937ccc79bb5be16": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HBoxModel",
@@ -21927,16 +22005,16 @@
        "_view_name": "HBoxView",
        "box_style": "",
        "children": [
-        "IPY_MODEL_7daabc68020641e897d4809176829307",
-        "IPY_MODEL_ba3727d9eb8e45b394738da6078d874c",
-        "IPY_MODEL_ac60beb4421b4e758aec659715fa5146"
+        "IPY_MODEL_cb4cbc4865294ceca967b149417812dd",
+        "IPY_MODEL_9cf230ec0aaf4f5590a6afd2c086e8a7",
+        "IPY_MODEL_a4de44337e914415b93595097425f7ac"
        ],
-       "layout": "IPY_MODEL_97133900a1a94bc691ce291364fca64c",
+       "layout": "IPY_MODEL_c47c55e36b944e7f9945ca4e513889f8",
        "tabbable": null,
        "tooltip": null
       }
      },
-     "5fe7263a54b34f2b918ca2c57d194d74": {
+     "5e99fedf15684824abf596d6692378e1": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -21989,7 +22067,7 @@
        "width": null
       }
      },
-     "621583ed869b478d913fc04a87073a95": {
+     "6bbb6e62a049423bb1bb4a903cd0f7ef": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -22042,7 +22120,7 @@
        "width": null
       }
      },
-     "6ab9eaf5ee3342379119468f880a7e68": {
+     "708a6bc613444a518f2b42ae989a958a": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "ProgressStyleModel",
@@ -22058,49 +22136,84 @@
        "description_width": ""
       }
      },
-     "6b5428d552484be2b498aa06f2cd12d7": {
+     "7473a57e0b514125b34860321843604d": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "FloatProgressModel",
+      "model_name": "HBoxModel",
       "state": {
        "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "FloatProgressModel",
+       "_model_name": "HBoxModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "ProgressView",
-       "bar_style": "success",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_488e5f543a8f48a79b17dd90afc28284",
-       "max": 1042301.0,
-       "min": 0.0,
-       "orientation": "horizontal",
-       "style": "IPY_MODEL_7ffcda766f894799bb645944b5ecab6b",
+       "_view_name": "HBoxView",
+       "box_style": "",
+       "children": [
+        "IPY_MODEL_2bf41dd86aaf412b8d50d2cf2ab37cad",
+        "IPY_MODEL_aa6a0d5ee722417d9d0a0c721f6a32bf",
+        "IPY_MODEL_2dae86562a8d47b5a5c7935ebcdfea75"
+       ],
+       "layout": "IPY_MODEL_d4bda90d6e2843c98dc87c59da4c233e",
        "tabbable": null,
-       "tooltip": null,
-       "value": 1042301.0
+       "tooltip": null
       }
      },
-     "71480013d795482ea7827866fd7d1fb0": {
-      "model_module": "@jupyter-widgets/controls",
+     "75f267512b70424083e6f6741d3132e0": {
+      "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
+      "model_name": "LayoutModel",
       "state": {
-       "_model_module": "@jupyter-widgets/controls",
+       "_model_module": "@jupyter-widgets/base",
        "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
+       "_model_name": "LayoutModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
+       "_view_name": "LayoutView",
+       "align_content": null,
+       "align_items": null,
+       "align_self": null,
+       "border_bottom": null,
+       "border_left": null,
+       "border_right": null,
+       "border_top": null,
+       "bottom": null,
+       "display": null,
+       "flex": null,
+       "flex_flow": null,
+       "grid_area": null,
+       "grid_auto_columns": null,
+       "grid_auto_flow": null,
+       "grid_auto_rows": null,
+       "grid_column": null,
+       "grid_gap": null,
+       "grid_row": null,
+       "grid_template_areas": null,
+       "grid_template_columns": null,
+       "grid_template_rows": null,
+       "height": null,
+       "justify_content": null,
+       "justify_items": null,
+       "left": null,
+       "margin": null,
+       "max_height": null,
+       "max_width": null,
+       "min_height": null,
+       "min_width": null,
+       "object_fit": null,
+       "object_position": null,
+       "order": null,
+       "overflow": null,
+       "padding": null,
+       "right": null,
+       "top": null,
+       "visibility": null,
+       "width": null
       }
      },
-     "7daabc68020641e897d4809176829307": {
+     "7d793895e94448fabb3f09887d5be895": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLModel",
@@ -22115,15 +22228,39 @@
        "_view_name": "HTMLView",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_3250ac219bb14a0d87380c723d7187c5",
+       "layout": "IPY_MODEL_46075fcab25949e4accbc84c39c7b694",
        "placeholder": "​",
-       "style": "IPY_MODEL_f6e7f731edb44410a1bbe854b20b75b3",
+       "style": "IPY_MODEL_dafd098f48e24be49b4261213ec00625",
        "tabbable": null,
        "tooltip": null,
-       "value": "tokenizer_config.json: 100%"
+       "value": "merges.txt: 100%"
+      }
+     },
+     "7fd3dba640c2447299e8235086a74547": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HBoxModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HBoxModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HBoxView",
+       "box_style": "",
+       "children": [
+        "IPY_MODEL_8f8c377a736d4e568f9f92ca89d5037d",
+        "IPY_MODEL_b2196b99196b44458834657ac7229487",
+        "IPY_MODEL_875219f2eae645a78d58982a80ba43d2"
+       ],
+       "layout": "IPY_MODEL_ae786b718eb746bf98a72ad65faeaa17",
+       "tabbable": null,
+       "tooltip": null
       }
      },
-     "7f602aa1ed364df1a417f0cf429e4a5c": {
+     "816c22a73aab42ab955422e98bcd057a": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLStyleModel",
@@ -22141,76 +22278,53 @@
        "text_color": null
       }
      },
-     "7ffcda766f894799bb645944b5ecab6b": {
+     "875219f2eae645a78d58982a80ba43d2": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
+      "model_name": "HTMLModel",
       "state": {
+       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
+       "_model_name": "HTMLModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
+       "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_058fdb1f616145a989119197863c8fa2",
+       "placeholder": "​",
+       "style": "IPY_MODEL_175ab33428e24054a9588b575b136309",
+       "tabbable": null,
+       "tooltip": null,
+       "value": " 124/124 [00:00&lt;00:00, 21.4kB/s]"
       }
      },
-     "853700c97ecf475a8336c57945c87c6a": {
-      "model_module": "@jupyter-widgets/base",
+     "896b626d27ed4333a78c8bc4b08fef82": {
+      "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "LayoutModel",
+      "model_name": "HTMLModel",
       "state": {
-       "_model_module": "@jupyter-widgets/base",
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "LayoutModel",
+       "_model_name": "HTMLModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
+       "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "LayoutView",
-       "align_content": null,
-       "align_items": null,
-       "align_self": null,
-       "border_bottom": null,
-       "border_left": null,
-       "border_right": null,
-       "border_top": null,
-       "bottom": null,
-       "display": null,
-       "flex": null,
-       "flex_flow": null,
-       "grid_area": null,
-       "grid_auto_columns": null,
-       "grid_auto_flow": null,
-       "grid_auto_rows": null,
-       "grid_column": null,
-       "grid_gap": null,
-       "grid_row": null,
-       "grid_template_areas": null,
-       "grid_template_columns": null,
-       "grid_template_rows": null,
-       "height": null,
-       "justify_content": null,
-       "justify_items": null,
-       "left": null,
-       "margin": null,
-       "max_height": null,
-       "max_width": null,
-       "min_height": null,
-       "min_width": null,
-       "object_fit": null,
-       "object_position": null,
-       "order": null,
-       "overflow": null,
-       "padding": null,
-       "right": null,
-       "top": null,
-       "visibility": null,
-       "width": null
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_75f267512b70424083e6f6741d3132e0",
+       "placeholder": "​",
+       "style": "IPY_MODEL_a8bd4932361949f2be691da2b7c4e787",
+       "tabbable": null,
+       "tooltip": null,
+       "value": " 548M/548M [00:02&lt;00:00, 201MB/s]"
       }
      },
-     "860eac54afd644cdb66576f10d1c4b94": {
+     "8a9ecc907dfa4a7b97bcd057b4f96110": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -22263,8 +22377,223 @@
        "width": null
       }
      },
-     "910235bb167a412ab2a54c930635b3bf": {
-      "model_module": "@jupyter-widgets/base",
+     "8f5ab92f87714abab2c7cce99a7ef76d": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "FloatProgressModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "FloatProgressModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "ProgressView",
+       "bar_style": "success",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_8a9ecc907dfa4a7b97bcd057b4f96110",
+       "max": 548105171.0,
+       "min": 0.0,
+       "orientation": "horizontal",
+       "style": "IPY_MODEL_063d9fb16c374bbcbdee214535e4af62",
+       "tabbable": null,
+       "tooltip": null,
+       "value": 548105171.0
+      }
+     },
+     "8f8c377a736d4e568f9f92ca89d5037d": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_f2554ae1c6064ee7aaa80c39701f6c74",
+       "placeholder": "​",
+       "style": "IPY_MODEL_816c22a73aab42ab955422e98bcd057a",
+       "tabbable": null,
+       "tooltip": null,
+       "value": "generation_config.json: 100%"
+      }
+     },
+     "91ad41831c3a4e5cadc2058781d8fb0d": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "ProgressStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "ProgressStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "bar_color": null,
+       "description_width": ""
+      }
+     },
+     "954188958f204f53bc21c9394f7c060c": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "ProgressStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "ProgressStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "bar_color": null,
+       "description_width": ""
+      }
+     },
+     "9bc7a1f74e244aeda156fded883d3fbc": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_1fd6a946b74140c8b91d0cfcc9c8a048",
+       "placeholder": "​",
+       "style": "IPY_MODEL_c4c48775aaab4649a5dc20fc1b976758",
+       "tabbable": null,
+       "tooltip": null,
+       "value": " 456k/456k [00:00&lt;00:00, 3.52MB/s]"
+      }
+     },
+     "9cf230ec0aaf4f5590a6afd2c086e8a7": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "FloatProgressModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "FloatProgressModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "ProgressView",
+       "bar_style": "success",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_00869a4f30534799a76cd92f9815d895",
+       "max": 1042301.0,
+       "min": 0.0,
+       "orientation": "horizontal",
+       "style": "IPY_MODEL_0a2dc7a44b50480498bd6e0a8258b228",
+       "tabbable": null,
+       "tooltip": null,
+       "value": 1042301.0
+      }
+     },
+     "a3a152183bb84d98b4545d9f13d5980b": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "a4de44337e914415b93595097425f7ac": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_f9181f78c4654800ada77429eb78dc13",
+       "placeholder": "​",
+       "style": "IPY_MODEL_220d51ccdafb40df864859c57034a447",
+       "tabbable": null,
+       "tooltip": null,
+       "value": " 1.04M/1.04M [00:00&lt;00:00, 5.31MB/s]"
+      }
+     },
+     "a8bd4932361949f2be691da2b7c4e787": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "aa6a0d5ee722417d9d0a0c721f6a32bf": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "FloatProgressModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "FloatProgressModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "ProgressView",
+       "bar_style": "success",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_44cc2a714dab4fbd92b45ec21a01bcef",
+       "max": 26.0,
+       "min": 0.0,
+       "orientation": "horizontal",
+       "style": "IPY_MODEL_91ad41831c3a4e5cadc2058781d8fb0d",
+       "tabbable": null,
+       "tooltip": null,
+       "value": 26.0
+      }
+     },
+     "ae786b718eb746bf98a72ad65faeaa17": {
+      "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
       "state": {
@@ -22316,25 +22645,33 @@
        "width": null
       }
      },
-     "924d96b2376c4b16a4541d64b0e45872": {
+     "b2196b99196b44458834657ac7229487": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
+      "model_name": "FloatProgressModel",
       "state": {
+       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
+       "_model_name": "FloatProgressModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
+       "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
+       "_view_name": "ProgressView",
+       "bar_style": "success",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_0f16d4bd6ea64fa6b925ab2efd1ca15e",
+       "max": 124.0,
+       "min": 0.0,
+       "orientation": "horizontal",
+       "style": "IPY_MODEL_449107685d8c439e8aaf9931f3acf205",
+       "tabbable": null,
+       "tooltip": null,
+       "value": 124.0
       }
      },
-     "9351dee6b05946a0a6880580b6f6ff6b": {
+     "baa17a1c01be427c8dd2fe2e9a46b3fc": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -22387,7 +22724,7 @@
        "width": null
       }
      },
-     "97133900a1a94bc691ce291364fca64c": {
+     "be0da02273224522ab03fcdf121a9d42": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -22440,97 +22777,7 @@
        "width": null
       }
      },
-     "9a499b42e58849dcbfc030ebb0630a62": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "FloatProgressModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "FloatProgressModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "ProgressView",
-       "bar_style": "success",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_4faadec4ac0f4a4ab32961fb3b74e529",
-       "max": 456318.0,
-       "min": 0.0,
-       "orientation": "horizontal",
-       "style": "IPY_MODEL_6ab9eaf5ee3342379119468f880a7e68",
-       "tabbable": null,
-       "tooltip": null,
-       "value": 456318.0
-      }
-     },
-     "aa33f2b3f7af4a5c956ebdbae08dabec": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_4973dab052f14376aa999fd6bc13197b",
-       "placeholder": "​",
-       "style": "IPY_MODEL_4064ed6431a14c8d940329269c2399e3",
-       "tabbable": null,
-       "tooltip": null,
-       "value": "tokenizer.json: 100%"
-      }
-     },
-     "ac60beb4421b4e758aec659715fa5146": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_c0e90bf5be684b4eadbd6f97b968ab31",
-       "placeholder": "​",
-       "style": "IPY_MODEL_e49babdcc96f4f99af8dba7cd03202fc",
-       "tabbable": null,
-       "tooltip": null,
-       "value": " 26.0/26.0 [00:00&lt;00:00, 4.60kB/s]"
-      }
-     },
-     "add5c8f3c40642ef8e4cf96579325ec3": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
-      }
-     },
-     "b1a0d471042a4a018d4de3bf29a3ecd2": {
+     "c47c55e36b944e7f9945ca4e513889f8": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -22583,23 +22830,25 @@
        "width": null
       }
      },
-     "b2dd202e54cc4c4bbf18779cef088e63": {
+     "c4c48775aaab4649a5dc20fc1b976758": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
+      "model_name": "HTMLStyleModel",
       "state": {
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
+       "_model_name": "HTMLStyleModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
        "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
       }
      },
-     "b4faa038d86840faa63b4ff7f8ac404c": {
+     "cb4cbc4865294ceca967b149417812dd": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLModel",
@@ -22614,15 +22863,15 @@
        "_view_name": "HTMLView",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_de2a35b1b3264fd2ac593af14da5b645",
+       "layout": "IPY_MODEL_32a2e286dd0e4b47a7ed286e57e25a02",
        "placeholder": "​",
-       "style": "IPY_MODEL_c02c62e902e648ef9e245988ad7917f6",
+       "style": "IPY_MODEL_19befccdb76441798bcd9c34f9a3970e",
        "tabbable": null,
        "tooltip": null,
-       "value": " 456k/456k [00:00&lt;00:00, 38.7MB/s]"
+       "value": "vocab.json: 100%"
       }
      },
-     "b60c8dc4072a47f69bc60e5115ab87c6": {
+     "d4bda90d6e2843c98dc87c59da4c233e": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -22658,73 +22907,24 @@
        "height": null,
        "justify_content": null,
        "justify_items": null,
-       "left": null,
-       "margin": null,
-       "max_height": null,
-       "max_width": null,
-       "min_height": null,
-       "min_width": null,
-       "object_fit": null,
-       "object_position": null,
-       "order": null,
-       "overflow": null,
-       "padding": null,
-       "right": null,
-       "top": null,
-       "visibility": null,
-       "width": null
-      }
-     },
-     "b9ffe90723d54a178c02c4a71079ef4f": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_9351dee6b05946a0a6880580b6f6ff6b",
-       "placeholder": "​",
-       "style": "IPY_MODEL_924d96b2376c4b16a4541d64b0e45872",
-       "tabbable": null,
-       "tooltip": null,
-       "value": "vocab.json: 100%"
-      }
-     },
-     "ba3727d9eb8e45b394738da6078d874c": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "FloatProgressModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "FloatProgressModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "ProgressView",
-       "bar_style": "success",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_853700c97ecf475a8336c57945c87c6a",
-       "max": 26.0,
-       "min": 0.0,
-       "orientation": "horizontal",
-       "style": "IPY_MODEL_47513102e87a47069219c229c4725eea",
-       "tabbable": null,
-       "tooltip": null,
-       "value": 26.0
+       "left": null,
+       "margin": null,
+       "max_height": null,
+       "max_width": null,
+       "min_height": null,
+       "min_width": null,
+       "object_fit": null,
+       "object_position": null,
+       "order": null,
+       "overflow": null,
+       "padding": null,
+       "right": null,
+       "top": null,
+       "visibility": null,
+       "width": null
       }
      },
-     "bec18b05c0a8413485c9f6fb6d8d51dd": {
+     "da86d7929cf44490b4235842717812d2": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -22777,7 +22977,7 @@
        "width": null
       }
      },
-     "bf988798a38d454fa0c118280dbeab56": {
+     "dafd098f48e24be49b4261213ec00625": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLStyleModel",
@@ -22795,25 +22995,31 @@
        "text_color": null
       }
      },
-     "c02c62e902e648ef9e245988ad7917f6": {
+     "e3f6e035556c4a02a0f9ccad87c2f138": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
+      "model_name": "HBoxModel",
       "state": {
+       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
+       "_model_name": "HBoxModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
+       "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
+       "_view_name": "HBoxView",
+       "box_style": "",
+       "children": [
+        "IPY_MODEL_48e6660d282a4e2db01422526059d280",
+        "IPY_MODEL_8f5ab92f87714abab2c7cce99a7ef76d",
+        "IPY_MODEL_896b626d27ed4333a78c8bc4b08fef82"
+       ],
+       "layout": "IPY_MODEL_41f933e12f1d4d79a42601d7b7c9da82",
+       "tabbable": null,
+       "tooltip": null
       }
      },
-     "c0e90bf5be684b4eadbd6f97b968ab31": {
+     "f2554ae1c6064ee7aaa80c39701f6c74": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -22866,30 +23072,7 @@
        "width": null
       }
      },
-     "c43b6d1b7dc04e8ea921c5940f890126": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_4a23001ae4824747ac51dbe487a0a126",
-       "placeholder": "​",
-       "style": "IPY_MODEL_add5c8f3c40642ef8e4cf96579325ec3",
-       "tabbable": null,
-       "tooltip": null,
-       "value": " 124/124 [00:00&lt;00:00, 24.5kB/s]"
-      }
-     },
-     "c655320743a4421684aca235370c46c1": {
+     "f2817ec73aab434a87ea26dff6fcd560": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -22942,31 +23125,7 @@
        "width": null
       }
      },
-     "c9ac7e38eb534678896df289998d1855": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HBoxModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HBoxModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HBoxView",
-       "box_style": "",
-       "children": [
-        "IPY_MODEL_b9ffe90723d54a178c02c4a71079ef4f",
-        "IPY_MODEL_6b5428d552484be2b498aa06f2cd12d7",
-        "IPY_MODEL_d8504344e45a40dba0505fafd0a4fef8"
-       ],
-       "layout": "IPY_MODEL_c655320743a4421684aca235370c46c1",
-       "tabbable": null,
-       "tooltip": null
-      }
-     },
-     "c9d427e21ae84197b7ff8cdb8bb4c89d": {
+     "f3b0ff1fd79f4f90800ffb7c4391e975": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "FloatProgressModel",
@@ -22982,128 +23141,17 @@
        "bar_style": "success",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_860eac54afd644cdb66576f10d1c4b94",
-       "max": 1355256.0,
+       "layout": "IPY_MODEL_f2817ec73aab434a87ea26dff6fcd560",
+       "max": 456318.0,
        "min": 0.0,
        "orientation": "horizontal",
-       "style": "IPY_MODEL_b2dd202e54cc4c4bbf18779cef088e63",
-       "tabbable": null,
-       "tooltip": null,
-       "value": 1355256.0
-      }
-     },
-     "cf0868984e46408fb4a1d566ec3db0b0": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HBoxModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HBoxModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HBoxView",
-       "box_style": "",
-       "children": [
-        "IPY_MODEL_aa33f2b3f7af4a5c956ebdbae08dabec",
-        "IPY_MODEL_c9d427e21ae84197b7ff8cdb8bb4c89d",
-        "IPY_MODEL_0dc2fdfccac3440ea6a358b4b727da08"
-       ],
-       "layout": "IPY_MODEL_bec18b05c0a8413485c9f6fb6d8d51dd",
-       "tabbable": null,
-       "tooltip": null
-      }
-     },
-     "d2847e7beeb442018670534e35889459": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
-      }
-     },
-     "d3bf56862e814e46a08ce4949ce5df67": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_621583ed869b478d913fc04a87073a95",
-       "placeholder": "​",
-       "style": "IPY_MODEL_d2847e7beeb442018670534e35889459",
-       "tabbable": null,
-       "tooltip": null,
-       "value": "merges.txt: 100%"
-      }
-     },
-     "d8504344e45a40dba0505fafd0a4fef8": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_4647f9260ab248d7a350e7a07c999a9b",
-       "placeholder": "​",
-       "style": "IPY_MODEL_7f602aa1ed364df1a417f0cf429e4a5c",
-       "tabbable": null,
-       "tooltip": null,
-       "value": " 1.04M/1.04M [00:00&lt;00:00, 25.8MB/s]"
-      }
-     },
-     "ddeeebc813c54775ab8841a88e512534": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_1dffcf97acf041dfa4829930fc637e56",
-       "placeholder": "​",
-       "style": "IPY_MODEL_bf988798a38d454fa0c118280dbeab56",
+       "style": "IPY_MODEL_954188958f204f53bc21c9394f7c060c",
        "tabbable": null,
        "tooltip": null,
-       "value": " 548M/548M [00:02&lt;00:00, 243MB/s]"
+       "value": 456318.0
       }
      },
-     "de2a35b1b3264fd2ac593af14da5b645": {
+     "f6926ea8116f41a487c8a794a0461d52": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -23156,43 +23204,7 @@
        "width": null
       }
      },
-     "e49babdcc96f4f99af8dba7cd03202fc": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
-      }
-     },
-     "ed04372d5ce8433d848d96e975723537": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
-      }
-     },
-     "f06d8ed2f2fc40c78e6f71a980042fa2": {
+     "f9181f78c4654800ada77429eb78dc13": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -23245,43 +23257,7 @@
        "width": null
       }
      },
-     "f6e7f731edb44410a1bbe854b20b75b3": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
-      }
-     },
-     "f988b1609264444ea2e348b491a028a4": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
-      }
-     },
-     "fe14fba431ee44719e467a94d38602c1": {
+     "f94c5abf57df4cdba73ed4b8c8bafb54": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -23333,6 +23309,30 @@
        "visibility": null,
        "width": null
       }
+     },
+     "fb98f7a2082142148838a198ceed048b": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HBoxModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HBoxModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HBoxView",
+       "box_style": "",
+       "children": [
+        "IPY_MODEL_7d793895e94448fabb3f09887d5be895",
+        "IPY_MODEL_f3b0ff1fd79f4f90800ffb7c4391e975",
+        "IPY_MODEL_9bc7a1f74e244aeda156fded883d3fbc"
+       ],
+       "layout": "IPY_MODEL_f6926ea8116f41a487c8a794a0461d52",
+       "tabbable": null,
+       "tooltip": null
+      }
      }
     },
     "version_major": 2,
diff --git a/generated/demos/Main_Demo.html b/generated/demos/Main_Demo.html
index 6700ea437..6fd77eed6 100644
--- a/generated/demos/Main_Demo.html
+++ b/generated/demos/Main_Demo.html
@@ -385,11 +385,11 @@ <h1>Setup<a class="headerlink" href="#Setup" title="Permalink to this heading">#
 </pre></div>
 </div>
 <div class="output_area rendered_html docutils container">
-<div id="circuits-vis-48c8c518-76ab" style="margin: 15px 0;"/>
+<div id="circuits-vis-88c92a02-5c72" style="margin: 15px 0;"/>
     <script crossorigin type="module">
     import { render, Hello } from "https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js";
     render(
-      "circuits-vis-48c8c518-76ab",
+      "circuits-vis-88c92a02-5c72",
       Hello,
       {"name": "Neel"}
     )
@@ -440,7 +440,7 @@ <h1>Setup<a class="headerlink" href="#Setup" title="Permalink to this heading">#
 </div>
 <div class="output_area docutils container">
 <div class="highlight"><pre>
-&lt;torch.autograd.grad_mode.set_grad_enabled at 0x7f0f50f9df90&gt;
+&lt;torch.autograd.grad_mode.set_grad_enabled at 0x7f027eb64110&gt;
 </pre></div></div>
 </div>
 <p>Plotting helper functions:</p>
@@ -593,11 +593,11 @@ <h2>Caching all Activations<a class="headerlink" href="#Caching-all-Activations"
 </pre></div>
 </div>
 <div class="output_area rendered_html docutils container">
-<div id="circuits-vis-ac0a9437-0fc4" style="margin: 15px 0;"/>
+<div id="circuits-vis-90f99dc5-665a" style="margin: 15px 0;"/>
     <script crossorigin type="module">
     import { render, AttentionPatterns } from "https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js";
     render(
-      "circuits-vis-ac0a9437-0fc4",
+      "circuits-vis-90f99dc5-665a",
       AttentionPatterns,
       {"tokens": ["<|endoftext|>", "Natural", " language", " processing", " tasks", ",", " such", " as", " question", " answering", ",", " machine", " translation", ",", " reading", " comprehension", ",", " and", " summar", "ization", ",", " are", " typically", " approached", " with", " supervised", " learning", " on", " tasks", "pe", "cific", " datasets", "."], "attention": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9639418125152588, 0.03605816140770912, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8389372825622559, 0.11828788369894028, 0.04277484491467476, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.47436124086380005, 0.13382026553153992, 0.27371731400489807, 0.11810113489627838, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.35606440901756287, 0.10184910148382187, 0.23054221272468567, 0.20397402346134186, 0.10757026076316833, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6660143733024597, 0.16866375505924225, 0.04535674676299095, 0.038855016231536865, 0.0677548199892044, 0.013355279341340065, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.38626962900161743, 0.285109281539917, 0.07609007507562637, 0.05908379331231117, 0.07223352044820786, 0.03979635238647461, 0.08141742646694183, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3775395154953003, 0.1883881539106369, 0.11723985522985458, 0.08685600012540817, 0.0666918158531189, 0.03500017523765564, 0.09693004935979843, 0.03135441616177559, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4869752824306488, 0.06781317293643951, 0.07952874153852463, 0.0848078578710556, 0.1590261608362198, 0.029577823355793953, 0.025685923174023628, 0.016474608331918716, 0.05011039599776268, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2906550168991089, 0.0401349738240242, 0.14614859223365784, 0.09940590709447861, 0.1538919061422348, 0.039001598954200745, 0.024988971650600433, 0.03184127062559128, 0.10222824662923813, 0.0717034563422203, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.39624103903770447, 0.09694176912307739, 0.027270659804344177, 0.02355135791003704, 0.03723449259996414, 0.006502415519207716, 0.08118756860494614, 0.013088458217680454, 0.06990590691566467, 0.24043099582195282, 0.007645336911082268, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.24864791333675385, 0.1380205899477005, 0.0923532024025917, 0.08676131069660187, 0.1381969302892685, 0.05914197862148285, 0.03223859518766403, 0.03158237785100937, 0.030489422380924225, 0.03873484209179878, 0.06671839207410812, 0.037114467471838, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1914844512939453, 0.1617259681224823, 0.07445938140153885, 0.07740949839353561, 0.021961113438010216, 0.03392130509018898, 0.05125021934509277, 0.01951923966407776, 0.03132446110248566, 0.04020152986049652, 0.038742680102586746, 0.21578852832317352, 0.042211681604385376, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3704318404197693, 0.08681417256593704, 0.02458467148244381, 0.021616315469145775, 0.032388731837272644, 0.005422737915068865, 0.0727522075176239, 0.0112727927044034, 0.06329693645238876, 0.21726809442043304, 0.006367155350744724, 0.029603807255625725, 0.05099846422672272, 0.007182050962001085, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1973765343427658, 0.04603995382785797, 0.0439998134970665, 0.1337345838546753, 0.05424821376800537, 0.025475721806287766, 0.027563506737351418, 0.021570932120084763, 0.051718227565288544, 0.06458097696304321, 0.02806464210152626, 0.23551592230796814, 0.019129814580082893, 0.029963519424200058, 0.021017681807279587, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08907236158847809, 0.01928834617137909, 0.1665353924036026, 0.07281264662742615, 0.04738640412688255, 0.024487905204296112, 0.028987322002649307, 0.019370367750525475, 0.026673022657632828, 0.07316635549068451, 0.025704585015773773, 0.04242359474301338, 0.058694612234830856, 0.028932694345712662, 0.18119072914123535, 0.09527372568845749, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2816729247570038, 0.06441289931535721, 0.018008552491664886, 0.01616962067782879, 0.023183872923254967, 0.0037532988935709, 0.05472247675061226, 0.007909760810434818, 0.046164702624082565, 0.16947267949581146, 0.004361647181212902, 0.021011337637901306, 0.0354907400906086, 0.004932564217597246, 0.0955522358417511, 0.14726325869560242, 0.005917454604059458, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.21305488049983978, 0.0591236874461174, 0.03382088243961334, 0.02747686579823494, 0.02839355543255806, 0.008422904647886753, 0.0400853231549263, 0.011629256419837475, 0.05295189097523689, 0.15404635667800903, 0.009831804782152176, 0.03610190004110336, 0.04737289249897003, 0.011069180443882942, 0.09972473233938217, 0.1397135704755783, 0.013185348361730576, 0.013994931243360043, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15871697664260864, 0.04387888312339783, 0.08712151646614075, 0.08998466283082962, 0.030738582834601402, 0.0341489352285862, 0.024917256087064743, 0.03139195218682289, 0.024823859333992004, 0.019790329039096832, 0.03625484183430672, 0.020694412291049957, 0.042840685695409775, 0.03820899501442909, 0.06234658882021904, 0.10919700562953949, 0.0413760244846344, 0.04916759952902794, 0.054400913417339325, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10485509783029556, 0.12122292071580887, 0.0648748055100441, 0.08768714964389801, 0.03434053063392639, 0.01748395338654518, 0.034151818603277206, 0.015289156697690487, 0.023312119767069817, 0.028306512162089348, 0.01872047781944275, 0.028111934661865234, 0.041905295103788376, 0.0209895521402359, 0.04678506404161453, 0.08659638464450836, 0.023631852120161057, 0.024273162707686424, 0.16702400147914886, 0.010438223369419575, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.22901973128318787, 0.05184381827712059, 0.013585173524916172, 0.012337238527834415, 0.018005413934588432, 0.0027703619562089443, 0.042381297796964645, 0.005856257397681475, 0.03614485636353493, 0.13039222359657288, 0.0031534270383417606, 0.01567256823182106, 0.027800394222140312, 0.003554322523996234, 0.07460816204547882, 0.11298283189535141, 0.004272268619388342, 0.006832208018749952, 0.18569746613502502, 0.018073637038469315, 0.005016352981328964, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.18869927525520325, 0.03438714146614075, 0.022344734519720078, 0.019972724840044975, 0.016354026272892952, 0.0068565551191568375, 0.020859038457274437, 0.005695997271686792, 0.03415916487574577, 0.07260986417531967, 0.007857211865484715, 0.018040239810943604, 0.026904456317424774, 0.009020394645631313, 0.06876447051763535, 0.17578734457492828, 0.010720067657530308, 0.00928453542292118, 0.1925639659166336, 0.025180332362651825, 0.012639074586331844, 0.021299341693520546, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1195831298828125, 0.022259412333369255, 0.032947149127721786, 0.020170224830508232, 0.035653311759233475, 0.013459913432598114, 0.017516475170850754, 0.010057873092591763, 0.025856440886855125, 0.05955953523516655, 0.015084508806467056, 0.015008730813860893, 0.053174685686826706, 0.016597608104348183, 0.041555255651474, 0.131293386220932, 0.01929667219519615, 0.015855036675930023, 0.1792508363723755, 0.01618383638560772, 0.02229553461074829, 0.015463392250239849, 0.10187702625989914, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14615531265735626, 0.026727521792054176, 0.01662455126643181, 0.018987685441970825, 0.06278638541698456, 0.015317163430154324, 0.019792240113019943, 0.01422776561230421, 0.025458168238401413, 0.045303549617528915, 0.016364356502890587, 0.037493038922548294, 0.0132886478677392, 0.017496541142463684, 0.0399458184838295, 0.05881758779287338, 0.01926097832620144, 0.024616025388240814, 0.038219697773456573, 0.02157779224216938, 0.02094990760087967, 0.07973217219114304, 0.05017608404159546, 0.17068105936050415, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11321667581796646, 0.03674636408686638, 0.011786575429141521, 0.010274871252477169, 0.020370664075016975, 0.00524388812482357, 0.015918847173452377, 0.005266785155981779, 0.02489173412322998, 0.06593258678913116, 0.005933663807809353, 0.018209032714366913, 0.021020200103521347, 0.0066674984991550446, 0.034828800708055496, 0.13742128014564514, 0.007927048951387405, 0.008618664927780628, 0.1137719452381134, 0.013557437807321548, 0.009277833625674248, 0.026121344417333603, 0.08499342203140259, 0.19073909521102905, 0.011263742111623287, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13337711989879608, 0.026216603815555573, 0.03827154263854027, 0.07152578234672546, 0.05317767709493637, 0.013925841078162193, 0.007084188051521778, 0.013450146652758121, 0.009841442108154297, 0.011789779178798199, 0.013537583872675896, 0.038154955953359604, 0.041933052241802216, 0.013882285915315151, 0.03707146272063255, 0.138384610414505, 0.014846334233880043, 0.03156952187418938, 0.05598173290491104, 0.015536684542894363, 0.01595635898411274, 0.045455560088157654, 0.01669965498149395, 0.02532576024532318, 0.03671892359852791, 0.08028541505336761, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10608974099159241, 0.019107727333903313, 0.024468660354614258, 0.027496395632624626, 0.016365809366106987, 0.0050114234909415245, 0.010413105599582195, 0.006081144325435162, 0.005301064345985651, 0.011143166571855545, 0.004565386101603508, 0.01896991953253746, 0.004321120213717222, 0.00481497822329402, 0.02940940298140049, 0.028682034462690353, 0.005097254645079374, 0.0072343479841947556, 0.03412593528628349, 0.010370595380663872, 0.005643266253173351, 0.007283586077392101, 0.029389560222625732, 0.010038800537586212, 0.009134513325989246, 0.546663224697113, 0.012777911499142647, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10467305034399033, 0.03321940079331398, 0.015341250225901604, 0.009373549371957779, 0.026595456525683403, 0.0057878089137375355, 0.013571349903941154, 0.004554884973913431, 0.028058893978595734, 0.026107225567102432, 0.006353434175252914, 0.013315831311047077, 0.026628265157341957, 0.006888872943818569, 0.06204749271273613, 0.05890703946352005, 0.008068050257861614, 0.007557098753750324, 0.0852278545498848, 0.017075754702091217, 0.009256886318325996, 0.019695758819580078, 0.12617813050746918, 0.13061513006687164, 0.011351036839187145, 0.08984372764825821, 0.04638150706887245, 0.007325289770960808, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08561903238296509, 0.021438946947455406, 0.056412599980831146, 0.0570666640996933, 0.019802208989858627, 0.00672749662771821, 0.005809164140373468, 0.004516261164098978, 0.0031647481955587864, 0.01761520653963089, 0.006174598820507526, 0.08767975866794586, 0.012299864552915096, 0.006350455805659294, 0.017522143200039864, 0.14295215904712677, 0.006585489492863417, 0.007875688374042511, 0.030078941956162453, 0.01390750240534544, 0.0073766945861279964, 0.007684784941375256, 0.02216084487736225, 0.012385032139718533, 0.011890063062310219, 0.08669772744178772, 0.19902527332305908, 0.01359447930008173, 0.029586246237158775, 0.0, 0.0, 0.0, 0.0], [0.14064587652683258, 0.013298707082867622, 0.015702949836850166, 0.017357859760522842, 0.02233150415122509, 0.029672270640730858, 0.04172082245349884, 0.01899542473256588, 0.03827710077166557, 0.048635613173246384, 0.03094690665602684, 0.016023898497223854, 0.020880894735455513, 0.032438233494758606, 0.030558118596673012, 0.022808339446783066, 0.03537759929895401, 0.031451594084501266, 0.03497113287448883, 0.01867910660803318, 0.03821907937526703, 0.02257886528968811, 0.06819561868906021, 0.04214095324277878, 0.028620855882763863, 0.03775003179907799, 0.018578052520751953, 0.03376871719956398, 0.03641696274280548, 0.012956855818629265, 0.0, 0.0, 0.0], [0.07168620079755783, 0.06924446672201157, 0.019306905567646027, 0.014161979779601097, 0.016823193058371544, 0.0193806029856205, 0.019257402047514915, 0.0220036581158638, 0.013706518337130547, 0.03578377887606621, 0.018465185537934303, 0.05207168683409691, 0.020085178315639496, 0.01986212469637394, 0.02066212333738804, 0.04725164547562599, 0.021076705306768417, 0.036787249147892, 0.024324089288711548, 0.0038275483530014753, 0.023920675739645958, 0.008533227257430553, 0.026241622865200043, 0.027380064129829407, 0.03461199253797531, 0.022884156554937363, 0.10047907382249832, 0.06913496553897858, 0.025474581867456436, 0.06495600938796997, 0.03061538189649582, 0.0, 0.0], [0.0691077709197998, 0.0370122492313385, 0.03862115740776062, 0.05933326855301857, 0.015923550352454185, 0.007918563671410084, 0.010371048003435135, 0.0066156634129583836, 0.002520075300708413, 0.026019366458058357, 0.007905230857431889, 0.02965201810002327, 0.04000623896718025, 0.008451344445347786, 0.010741154663264751, 0.05027563124895096, 0.009428859688341618, 0.013601033948361874, 0.05036921799182892, 0.03176714852452278, 0.010793027468025684, 0.007216801401227713, 0.006478427443653345, 0.01480061188340187, 0.021585967391729355, 0.15769484639167786, 0.08884759992361069, 0.0190169308334589, 0.02972934953868389, 0.03316137194633484, 0.050883661955595016, 0.034150850027799606, 0.0], [0.14375509321689606, 0.01681104488670826, 0.009386662393808365, 0.006830313708633184, 0.011656845919787884, 0.0015672279987484217, 0.019711481407284737, 0.002398042706772685, 0.021235886961221695, 0.04683680087327957, 0.0016905148513615131, 0.0058271917514503, 0.011979999952018261, 0.0018251396249979734, 0.042313333600759506, 0.054913729429244995, 0.0021786566358059645, 0.0024170097894966602, 0.09604065865278244, 0.005752880126237869, 0.0025577889755368233, 0.0071212234906852245, 0.08889931440353394, 0.10852081328630447, 0.005179054103791714, 0.03657734394073486, 0.02471994049847126, 0.0037347625475376844, 0.031077096238732338, 0.016887947916984558, 0.09450862556695938, 0.07171519845724106, 0.0033723204396665096]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0004246663593221456, 0.9995753169059753, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0005621908348985016, 0.016407281160354614, 0.9830306172370911, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0011627554195001721, 0.021681973710656166, 0.0037620372604578733, 0.9733933210372925, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [3.7244368286337703e-05, 0.00017202268645633012, 0.0002814392792060971, 0.0027421435806900263, 0.9967671632766724, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00826845783740282, 0.00023985625011846423, 7.361917960224673e-05, 6.43773382762447e-05, 0.0001756635756464675, 0.9911779761314392, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0012215041788294911, 0.005400444380939007, 0.0016716312384232879, 0.00040775578236207366, 0.0006163655198179185, 0.001093117636628449, 0.989589273929596, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0012459794525057077, 0.0009121228358708322, 0.0005976726533845067, 0.00013656896771863103, 0.00033041107235476375, 0.001572280889376998, 0.0038808276876807213, 0.9913240671157837, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00028217516955919564, 0.004068182315677404, 0.0026605194434523582, 0.0013093105517327785, 0.008030476048588753, 0.00028790938085876405, 0.00022922940843272954, 0.0003948427038267255, 0.9827372431755066, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [3.47392306139227e-05, 0.0003953832492697984, 0.00013272685464471579, 0.0002585221081972122, 0.001085555530153215, 9.198026964440942e-05, 0.000326707202475518, 0.000542744412086904, 0.006105918437242508, 0.9910256266593933, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0033785062842071056, 5.0908744015032426e-05, 1.6452069758088328e-05, 1.69261602422921e-05, 4.18141353293322e-05, 0.49394020438194275, 0.00012981283362023532, 0.0008837337954901159, 3.221205042791553e-05, 2.7252061045146547e-05, 0.5014821887016296, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [8.416055788984522e-05, 0.0013417234877124429, 0.001261359779164195, 0.0021450764033943415, 0.004042366519570351, 0.00048305385280400515, 0.0001158266604761593, 0.00015203595103230327, 2.6925330530502833e-05, 0.00012675137259066105, 0.00031289938488043845, 0.9899077415466309, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0003782230196520686, 0.0009837561519816518, 0.03934125974774361, 0.002732245484367013, 0.0036680190823972225, 0.00011039189848816022, 0.00012931032688356936, 0.00021743457182310522, 0.00010623293928802013, 0.0007748182397335768, 6.647665577474982e-05, 0.0003148667747154832, 0.9511768817901611, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0021017189137637615, 2.443686025799252e-05, 7.78878529672511e-06, 8.651618372823577e-06, 2.014001256611664e-05, 0.2997134327888489, 7.525274122599512e-05, 0.0004898307379335165, 1.845947736001108e-05, 1.534453986096196e-05, 0.32833898067474365, 4.175798676442355e-05, 6.469185791502241e-06, 0.3691376745700836, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0001496832846896723, 0.00011296618322376162, 0.00036294886376708746, 0.00018591222760733217, 0.00016460890765301883, 4.1432296711718664e-05, 2.8764745366061106e-05, 7.786958303768188e-05, 0.0009200992644764483, 0.0103401904925704, 2.7572339604375884e-05, 1.7833237507147714e-05, 0.0003305452992208302, 2.4375704015255906e-05, 0.9872152805328369, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00010753834794741124, 0.0021781916730105877, 0.002042611828073859, 0.004251930397003889, 0.006989913992583752, 2.511881393729709e-05, 0.0007779046427458525, 0.0005783551605418324, 0.00293784704990685, 0.033225417137145996, 1.71992760442663e-05, 0.0008936497615650296, 0.0015238587511703372, 1.4656818166258745e-05, 0.00622264388948679, 0.9382131695747375, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.001363488263450563, 1.3226085684436839e-05, 4.013950274384115e-06, 4.803066076419782e-06, 1.0257452231599018e-05, 0.19665955007076263, 4.527267810772173e-05, 0.0002776262117549777, 1.1714444553945214e-05, 9.473311365582049e-06, 0.22919613122940063, 2.6430649086250924e-05, 4.101847935089609e-06, 0.26576095819473267, 8.515355148119852e-06, 4.536029791779583e-06, 0.30659976601600647, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0008924751309677958, 0.00013490811397787184, 4.779841037816368e-05, 5.8037036069436e-05, 0.00010480164928594604, 0.012799090705811977, 0.0007168247830122709, 0.032579511404037476, 2.6449959477758966e-05, 0.00011185064795427024, 0.011884261853992939, 4.010266638942994e-05, 5.555404641199857e-05, 0.012377863749861717, 0.00010783471225295216, 5.4043663112679496e-05, 0.013122126460075378, 0.914886474609375, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [5.075309672974981e-06, 4.396129952510819e-05, 3.398501576157287e-05, 7.940301293274388e-05, 5.477911327034235e-05, 7.921535711830074e-07, 9.313323971582577e-06, 7.727071533736307e-06, 8.597254054620862e-05, 0.00012274031178094447, 5.141479277881444e-07, 1.702793611002562e-06, 3.83417536795605e-05, 4.4509667418424215e-07, 0.00013928221596870571, 0.0003275797644164413, 3.9947968843989656e-07, 3.948126504838001e-06, 0.999043881893158, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [9.346180740976706e-05, 0.0018395738443359733, 0.0025233125779777765, 0.018087238073349, 0.0029363748617470264, 0.00027335810591466725, 4.872974750469439e-05, 0.0004212774510961026, 0.00015624347724951804, 0.0009748333832249045, 0.00020533586211968213, 0.0010228825267404318, 0.0019548034761101007, 0.0001947036653291434, 0.0011294205905869603, 0.0016656103543937206, 0.00018734042532742023, 0.0009503448964096606, 0.0004455175076145679, 0.964889645576477, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0010859702015295625, 8.512333806720562e-06, 2.5307288069598144e-06, 3.0625205909018405e-06, 5.975129170110449e-06, 0.13292615115642548, 3.345326331327669e-05, 0.00018891248328145593, 8.477756637148559e-06, 6.540415142808342e-06, 0.1644592434167862, 1.813003655115608e-05, 3.0627427349827485e-06, 0.19719170033931732, 6.4298355937353335e-06, 3.444739604674396e-06, 0.23317119479179382, 0.0022796259727329016, 3.713432988661225e-06, 3.52880269929301e-05, 0.2685585916042328, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0004197956877760589, 0.00011805014219135046, 0.00014240501332096756, 3.796788223553449e-05, 0.00019043161591980606, 0.00176512252073735, 0.00057098304387182, 0.0005008855368942022, 8.840763621265069e-05, 0.0001420867774868384, 0.001663984265178442, 3.348111204104498e-05, 2.4413982828264125e-05, 0.00175465049687773, 6.520311580970883e-05, 2.414266964478884e-05, 0.0018299149814993143, 0.0015691040316596627, 3.974881110480055e-05, 0.00015712414460722357, 0.0018554475391283631, 0.9870065450668335, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [7.501777145080268e-05, 0.0014379826607182622, 6.345151632558554e-05, 0.00010864829528145492, 0.00015633167640771717, 3.2101231681735953e-06, 0.002203279873356223, 0.00022076554887462407, 5.240297468844801e-05, 4.8815789341460913e-05, 2.2644592263532104e-06, 1.5327233995776623e-05, 4.157140665483894e-06, 2.022839225901407e-06, 6.296796982496744e-06, 4.848669050261378e-05, 1.9928991150663933e-06, 3.247004497097805e-05, 0.0012695090845227242, 1.9632627299870364e-05, 1.8090934190695407e-06, 0.0005810288130305707, 0.9936450719833374, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [7.450151315424591e-05, 0.0006139386096037924, 0.0009361191187053919, 0.0008487795130349696, 0.002850631484761834, 1.0365051821281668e-05, 0.00021614256547763944, 0.00017397530609741807, 0.0020508402958512306, 0.005805297289043665, 8.055229955061805e-06, 8.086592424660921e-05, 0.0007702436414547265, 7.28818440620671e-06, 0.0010576159693300724, 0.002275596372783184, 6.663255135208601e-06, 0.00011621018347796053, 0.0005972451181150973, 8.736297604627907e-05, 6.332331849989714e-06, 6.096452852943912e-05, 6.090577517170459e-05, 0.9812840819358826, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0005848742439411581, 0.00015909200010355562, 1.083647021005163e-05, 7.365475903498009e-05, 0.00011349534906912595, 0.0008256662986241281, 0.00031911072437651455, 0.018529996275901794, 1.0226598533336073e-05, 4.9587219109525904e-05, 0.0007716424879617989, 4.454819281818345e-05, 9.865034371614456e-06, 0.0008067170856520534, 2.2673882995150052e-05, 1.2464139217627235e-05, 0.0008449103333987296, 0.008790111169219017, 3.5792407288681716e-05, 3.66286258213222e-05, 0.0008917527738958597, 0.0010791773675009608, 0.0003708462754730135, 0.0001083713723346591, 0.9654979109764099, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [1.6459925973322242e-05, 0.00017152438522316515, 3.2083211408462375e-05, 0.00010234172805212438, 0.002631882205605507, 9.886184670904186e-06, 3.250848749303259e-05, 3.7417539715534076e-05, 0.00012631539721041918, 4.9912112444872037e-05, 8.302540663862601e-06, 8.443430851912126e-05, 3.127968739136122e-05, 7.633363566128537e-06, 1.0101352927449625e-05, 5.6673809012863785e-05, 7.442129117407603e-06, 2.7689655325957574e-05, 1.841835728555452e-05, 2.879437261071871e-06, 6.840427886345424e-06, 4.279875156498747e-06, 0.00043176551116630435, 0.0001761750172590837, 8.995599637273699e-05, 0.9958257675170898, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [2.6368099497631192e-05, 0.00018008743063546717, 0.0001808296947274357, 0.00030464722658507526, 0.00039390946039929986, 4.674849333241582e-05, 2.7231719286646694e-05, 4.8734953452367336e-05, 0.00029131697374396026, 0.0004206168814562261, 3.804643711191602e-05, 0.0002524516312405467, 5.6067383411573246e-05, 3.82037615054287e-05, 0.0015365129802376032, 0.001253720954991877, 3.593428846215829e-05, 2.303666406078264e-05, 0.0001803626073524356, 0.0001226610183948651, 3.517790537443943e-05, 6.924685294507071e-05, 0.00011267305671935901, 0.0008507389575242996, 0.0001436190214008093, 0.00023528003657702357, 0.9930958151817322, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0007048255065456033, 5.675383727066219e-05, 3.351289251440903e-06, 7.472657671314664e-06, 1.785946005838923e-05, 0.0008156524272635579, 4.9529528041603044e-05, 0.0013530971482396126, 3.741759792319499e-05, 0.00014378006744664162, 0.0007639332907274365, 9.991676051868126e-06, 2.8844410735473502e-06, 0.0007552222232334316, 0.00010159592784475535, 3.1205443065118743e-06, 0.000806094438303262, 0.0011406756239011884, 1.4343090697366279e-05, 9.910167136695236e-06, 0.0008649186929687858, 8.663265907671303e-05, 3.4695403883233666e-05, 0.00010265821038046852, 0.007663471158593893, 4.187891317997128e-05, 3.4908698580693454e-06, 0.9844048619270325, 0.0, 0.0, 0.0, 0.0, 0.0], [2.0288332962081768e-05, 2.954272713395767e-05, 5.0375034334138036e-05, 0.00097783247474581, 0.3728252649307251, 6.6613192757358775e-06, 1.573364352225326e-05, 3.981243571615778e-05, 0.00022353240638040006, 0.00012674322351813316, 5.118445187690668e-06, 0.00024116165877785534, 1.2973839147889521e-05, 4.8006963879743125e-06, 2.3321663320530206e-05, 7.723527232883498e-05, 4.59344209957635e-06, 1.9647124645416625e-05, 0.00021129964443389326, 1.1453522347437683e-05, 4.380489826871781e-06, 1.4442671272263397e-05, 3.676743290270679e-05, 0.00011845333210658282, 3.7977195461280644e-05, 0.0007802762556821108, 0.00040483634802512825, 1.042955591401551e-05, 0.6236649751663208, 0.0, 0.0, 0.0, 0.0], [2.0582927390933037e-05, 8.831251761876047e-05, 0.00020454880723264068, 0.0003018901334144175, 8.223887562053278e-05, 1.9704415535670705e-05, 0.00014085126167628914, 2.896317164413631e-05, 7.669385013286956e-06, 3.7246169085847214e-05, 1.673677070357371e-05, 6.404696614481509e-05, 0.0006910591037012637, 1.6027584933908656e-05, 0.00015603694191668183, 0.00014825927792117, 1.5700084986747243e-05, 9.155390580417588e-05, 8.525098382961005e-05, 4.904517481918447e-06, 1.5784396964590997e-05, 5.293117283144966e-05, 0.000529835931956768, 0.0005658396985381842, 6.167324318084866e-05, 6.729497545165941e-05, 0.0003077442815992981, 1.0369159099354874e-05, 6.799342372687533e-05, 0.9960988759994507, 0.0, 0.0, 0.0], [1.3371318345889449e-05, 0.0009821535786613822, 0.0004154810740146786, 0.0001144233756349422, 0.00038730789674445987, 5.660860551870428e-06, 0.0012746280990540981, 0.0005708065000362694, 0.0006383630097843707, 0.0005776663310825825, 4.127733518544119e-06, 9.16175213205861e-06, 9.142841736320406e-05, 3.774867764150258e-06, 1.3575321645475924e-05, 0.0002916179655585438, 3.4474890071578557e-06, 7.899192860350013e-05, 0.003189122537150979, 4.885083853878314e-06, 3.165286670991918e-06, 1.4087469025980681e-05, 0.0001567144354339689, 0.0003544889041222632, 0.00017265471979044378, 0.0013050634879618883, 0.00021867688337806612, 2.6776719096233137e-05, 0.00026460207300260663, 1.3334529285202734e-05, 0.9888005256652832, 0.0, 0.0], [2.745506208157167e-05, 0.00016438262537121773, 7.99642366473563e-05, 0.001191496616229415, 0.0007883630460128188, 2.658414359757444e-06, 3.005756479979027e-05, 7.4579688771336805e-06, 0.00014940995606593788, 2.8857133656856604e-05, 1.873827500276093e-06, 0.00033288003760389984, 5.160855653230101e-05, 1.7577448261363315e-06, 0.0001265659084310755, 0.00014267012011259794, 1.6954454622464254e-06, 2.1952573661110364e-05, 0.00023040804080665112, 4.429338878253475e-05, 1.6103306279546814e-06, 2.7008050892618485e-05, 0.00023884844267740846, 0.00019046018132939935, 9.496136954112444e-06, 0.000446643796749413, 0.00022095769236329943, 5.379170943342615e-06, 0.0006956399301998317, 0.0001547076681163162, 0.0002548544143792242, 0.9943286776542664, 0.0], [0.006231049541383982, 9.72282505244948e-05, 6.871603090985445e-06, 2.1151156033738516e-05, 5.8280591474613175e-05, 0.007238905411213636, 2.0987936295568943e-05, 0.00025459096650592983, 6.243876850930974e-05, 2.0924684577039443e-05, 0.007872147485613823, 5.5853353842394426e-05, 9.868796041700989e-06, 0.009169402532279491, 7.203016866696998e-05, 7.068700597301358e-06, 0.010345976799726486, 0.0013096537441015244, 3.803684376180172e-05, 8.022711699595675e-05, 0.012053261511027813, 4.071998773724772e-05, 3.6860749332845444e-06, 3.4713455534074455e-05, 0.0005061827832832932, 8.918941603042185e-05, 2.9112143238307908e-05, 0.0012772815534844995, 8.489656465826556e-05, 0.00018447409092914313, 0.00013425754150375724, 6.813067739130929e-05, 0.9425214529037476]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.943029522895813, 0.05697042867541313, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9185556173324585, 0.03280005604028702, 0.048644352704286575, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8779287934303284, 0.056434255093336105, 0.04271192103624344, 0.02292500250041485, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.804131805896759, 0.029098201543092728, 0.075567327439785, 0.05643591657280922, 0.03476677089929581, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4943104684352875, 0.02018355205655098, 0.027966560795903206, 0.01831907220184803, 0.03144203498959541, 0.40777823328971863, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6057478785514832, 0.02924242615699768, 0.09491512179374695, 0.07609347254037857, 0.06614658236503601, 0.08705786615610123, 0.0407966710627079, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4483822286128998, 0.04542430862784386, 0.0740148276090622, 0.06864846497774124, 0.09376619011163712, 0.0877426490187645, 0.06534270942211151, 0.11667861044406891, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.49169260263442993, 0.13782073557376862, 0.03955018147826195, 0.06153320148587227, 0.04539964720606804, 0.04073145240545273, 0.06228705495595932, 0.05861866846680641, 0.062366463243961334, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5404124855995178, 0.044426653534173965, 0.03957854211330414, 0.04188809171319008, 0.07529856264591217, 0.04669506475329399, 0.048475231975317, 0.05500520393252373, 0.08293062448501587, 0.02528950944542885, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.28273555636405945, 0.014234175905585289, 0.017647748813033104, 0.011433064006268978, 0.021741706877946854, 0.2666538953781128, 0.015403537079691887, 0.047349266707897186, 0.017767544835805893, 0.013926065526902676, 0.2911074459552765, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.34496626257896423, 0.046116773039102554, 0.05771546810865402, 0.11131857335567474, 0.11289366334676743, 0.027930336073040962, 0.0385919027030468, 0.05656527727842331, 0.05864058807492256, 0.06648595631122589, 0.026114359498023987, 0.052660852670669556, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.46992170810699463, 0.032015684992074966, 0.1077289879322052, 0.027006765827536583, 0.04465881362557411, 0.022773560136556625, 0.0231170691549778, 0.025491517037153244, 0.049502693116664886, 0.026573937386274338, 0.01970885880291462, 0.06337957829236984, 0.08812081813812256, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.20661765336990356, 0.01037929393351078, 0.012261927127838135, 0.008319098502397537, 0.016007069498300552, 0.19527742266654968, 0.011453290469944477, 0.03475669026374817, 0.01307358592748642, 0.010938976891338825, 0.21602483093738556, 0.005866494961082935, 0.023642312735319138, 0.23538129031658173, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.33993491530418396, 0.029161635786294937, 0.09540709853172302, 0.03395187482237816, 0.08440458029508591, 0.012559544295072556, 0.02935866080224514, 0.024564165621995926, 0.10622433573007584, 0.04689216986298561, 0.011469592340290546, 0.006369193084537983, 0.11145274341106415, 0.011317990720272064, 0.0569315105676651, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4040880799293518, 0.024195658043026924, 0.038910072296857834, 0.014727434143424034, 0.024456558749079704, 0.038450006395578384, 0.03923036903142929, 0.037171367555856705, 0.06030004844069481, 0.041985444724559784, 0.037167176604270935, 0.01639123260974884, 0.0392896942794323, 0.03772977367043495, 0.13448578119277954, 0.011421292088925838, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15876448154449463, 0.008203906007111073, 0.0092952661216259, 0.0062217190861701965, 0.011793204583227634, 0.15069791674613953, 0.008851958438754082, 0.026313532143831253, 0.010186923667788506, 0.00843372568488121, 0.16762229800224304, 0.004420033656060696, 0.01821179874241352, 0.183508038520813, 0.020927347242832184, 0.006447790190577507, 0.20010006427764893, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.17539216578006744, 0.02056964486837387, 0.01829143613576889, 0.009298020042479038, 0.017377890646457672, 0.042534783482551575, 0.02070159651339054, 0.050443943589925766, 0.02543804794549942, 0.017218226566910744, 0.04311535507440567, 0.013349834829568863, 0.0285286046564579, 0.045972540974617004, 0.03408820927143097, 0.019834214821457863, 0.04992839694023132, 0.36791715025901794, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.26033347845077515, 0.017148371785879135, 0.03745276480913162, 0.07594798505306244, 0.04674699530005455, 0.018068527802824974, 0.031346458941698074, 0.037415146827697754, 0.07175809890031815, 0.05872536823153496, 0.0170787014067173, 0.04030593857169151, 0.05706355720758438, 0.01711357943713665, 0.1049133837223053, 0.046705346554517746, 0.017230207100510597, 0.024682143703103065, 0.019963975995779037, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.30547064542770386, 0.05190496891736984, 0.043468981981277466, 0.02184685505926609, 0.02101718634366989, 0.03390473872423172, 0.04190470278263092, 0.03909286484122276, 0.028871973976492882, 0.023003432899713516, 0.03205801919102669, 0.023334519937634468, 0.0711059421300888, 0.03290088474750519, 0.061641961336135864, 0.03183262050151825, 0.033767636865377426, 0.04571487754583359, 0.035015225410461426, 0.022141898050904274, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1224295049905777, 0.006105300039052963, 0.006670300383120775, 0.00458177737891674, 0.009337784722447395, 0.11381794512271881, 0.006783359684050083, 0.01971990056335926, 0.00758085772395134, 0.006613647099584341, 0.12765924632549286, 0.0035026604309678078, 0.01400233618915081, 0.14000985026359558, 0.015684885904192924, 0.005092614330351353, 0.15336214005947113, 0.03527415543794632, 0.02246563322842121, 0.006954458076506853, 0.17235167324543, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.20538486540317535, 0.03477850183844566, 0.014682911336421967, 0.03118330053985119, 0.030931269749999046, 0.021952766925096512, 0.03290865570306778, 0.05740531533956528, 0.05587516725063324, 0.04864277318120003, 0.02352074719965458, 0.015108555555343628, 0.02738633006811142, 0.02451845072209835, 0.060604583472013474, 0.034776147454977036, 0.026137804612517357, 0.051684360951185226, 0.06281405687332153, 0.020291464403271675, 0.02860172651708126, 0.0908101499080658, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.23978599905967712, 0.031323499977588654, 0.05037755146622658, 0.01586942747235298, 0.03901456296443939, 0.02280554547905922, 0.04285356029868126, 0.028882304206490517, 0.04046262055635452, 0.0341072604060173, 0.022644521668553352, 0.03923071548342705, 0.0723857581615448, 0.022345518693327904, 0.04958106949925423, 0.03193334490060806, 0.023325597867369652, 0.04521360620856285, 0.030554356053471565, 0.022876497358083725, 0.02473229542374611, 0.055494822561740875, 0.014199569821357727, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.23581692576408386, 0.020558306947350502, 0.04375005513429642, 0.02970486506819725, 0.03703876957297325, 0.014953454956412315, 0.04004311189055443, 0.027184365317225456, 0.0457618422806263, 0.03809259459376335, 0.014181883074343204, 0.03789152577519417, 0.06518243253231049, 0.014182924292981625, 0.05489495024085045, 0.023720961064100266, 0.014592787250876427, 0.025570029392838478, 0.07356183975934982, 0.039182331413030624, 0.014925522729754448, 0.04628865420818329, 0.027801064774394035, 0.015118853189051151, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14254964888095856, 0.012010160833597183, 0.016881300136446953, 0.020742561668157578, 0.032451752573251724, 0.029625998809933662, 0.030295221135020256, 0.05620869994163513, 0.029608748853206635, 0.02948184125125408, 0.03263983502984047, 0.010038234293460846, 0.04078620672225952, 0.03462785482406616, 0.03391636535525322, 0.02015574835240841, 0.036843158304691315, 0.0606469102203846, 0.047447122633457184, 0.032525286078453064, 0.040400370955467224, 0.05947763845324516, 0.03129400685429573, 0.04792547971010208, 0.07141980528831482, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2222907692193985, 0.059111371636390686, 0.03702675551176071, 0.04059014096856117, 0.0272544976323843, 0.019174763932824135, 0.03171538934111595, 0.02046213671565056, 0.038113273680210114, 0.019927890971302986, 0.018538912758231163, 0.015436704270541668, 0.04536491632461548, 0.01935577020049095, 0.050358302891254425, 0.03328138589859009, 0.02017974853515625, 0.03679436072707176, 0.04331387206912041, 0.028476405888795853, 0.02131732925772667, 0.047712888568639755, 0.013107037171721458, 0.02633604407310486, 0.030211349949240685, 0.03454800695180893, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.28438133001327515, 0.023870104923844337, 0.04641882702708244, 0.010260095819830894, 0.03390985727310181, 0.01830146834254265, 0.0237088892608881, 0.0247668344527483, 0.025767752900719643, 0.022968804463744164, 0.016735462471842766, 0.013406947255134583, 0.045986607670784, 0.01667322963476181, 0.08106391876935959, 0.05033260956406593, 0.016726160421967506, 0.019904790446162224, 0.03032534383237362, 0.01014632172882557, 0.01731823943555355, 0.019040953367948532, 0.01108111348003149, 0.052046384662389755, 0.033353518694639206, 0.03880883380770683, 0.012695519253611565, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11854357272386551, 0.01335094589740038, 0.013422034680843353, 0.030273202806711197, 0.02616293355822563, 0.020776210352778435, 0.021280204877257347, 0.03798932954668999, 0.03536444157361984, 0.036651790142059326, 0.022004006430506706, 0.01538003608584404, 0.030023131519556046, 0.023299671709537506, 0.03000757284462452, 0.014861369505524635, 0.02452193759381771, 0.03999800235033035, 0.03900156170129776, 0.036007679998874664, 0.026851212605834007, 0.06795871257781982, 0.038931310176849365, 0.057258062064647675, 0.0665983259677887, 0.03075685352087021, 0.023343220353126526, 0.059382691979408264, 0.0, 0.0, 0.0, 0.0, 0.0], [0.25091853737831116, 0.01135188713669777, 0.023033952340483665, 0.01884976588189602, 0.013208821415901184, 0.016430404037237167, 0.03763698413968086, 0.021519413217902184, 0.03823148459196091, 0.03122309409081936, 0.01608109660446644, 0.017179012298583984, 0.0823964774608612, 0.01577541045844555, 0.04969313368201256, 0.033979032188653946, 0.016469145193696022, 0.025656109675765038, 0.053269319236278534, 0.024380972608923912, 0.017032906413078308, 0.031062550842761993, 0.014834268018603325, 0.04310780018568039, 0.027818351984024048, 0.015127941034734249, 0.012749841436743736, 0.026852674782276154, 0.014129597693681717, 0.0, 0.0, 0.0, 0.0], [0.19256946444511414, 0.022833675146102905, 0.01449588406831026, 0.028055010363459587, 0.03290426358580589, 0.018577100709080696, 0.023776723071932793, 0.014988909475505352, 0.027755476534366608, 0.01995212957262993, 0.018426887691020966, 0.0268084853887558, 0.04026195779442787, 0.01895732805132866, 0.01998024806380272, 0.039050325751304626, 0.01949433982372284, 0.030714301392436028, 0.07932323962450027, 0.03619766980409622, 0.02037958987057209, 0.023319289088249207, 0.018723847344517708, 0.05692768841981888, 0.023927535861730576, 0.03924323245882988, 0.02178351581096649, 0.020374197512865067, 0.04178833216428757, 0.00840924121439457, 0.0, 0.0, 0.0], [0.1891939342021942, 0.01435206737369299, 0.027829350903630257, 0.0189360398799181, 0.05455230921506882, 0.024302387610077858, 0.0207525584846735, 0.03050178475677967, 0.01690041646361351, 0.029904041439294815, 0.023111775517463684, 0.02166486158967018, 0.03335980698466301, 0.023050500079989433, 0.02792702615261078, 0.026253484189510345, 0.0242579597979784, 0.024394970387220383, 0.017666669562458992, 0.02208106406033039, 0.025441503152251244, 0.023996729403734207, 0.015941407531499863, 0.021863479167222977, 0.04876275360584259, 0.01023405697196722, 0.025762980803847313, 0.06192327290773392, 0.06737110018730164, 0.01818411611020565, 0.009525515139102936, 0.0, 0.0], [0.27111950516700745, 0.05460572615265846, 0.039705973118543625, 0.03512895852327347, 0.02031376212835312, 0.0081838583573699, 0.022556638345122337, 0.01403643935918808, 0.02619846910238266, 0.032355356961488724, 0.007185023743659258, 0.010813490487635136, 0.05162610113620758, 0.007057540584355593, 0.04232776537537575, 0.01850598305463791, 0.006952997762709856, 0.012550847604870796, 0.037855181843042374, 0.014787373133003712, 0.007130768615752459, 0.01895672082901001, 0.01075897179543972, 0.023540528491139412, 0.01625620573759079, 0.007915404625236988, 0.03652312234044075, 0.014562270604074001, 0.019251469522714615, 0.010911807417869568, 0.0343967042863369, 0.06592899560928345, 0.0], [0.14836537837982178, 0.014506888575851917, 0.007646295242011547, 0.011038758791983128, 0.02630247175693512, 0.01319483295083046, 0.027184898033738136, 0.017392203211784363, 0.016122139990329742, 0.032880790531635284, 0.014050280675292015, 0.0071673463098704815, 0.016120055690407753, 0.014647023752331734, 0.018450895324349403, 0.013102126307785511, 0.01528322696685791, 0.01454298384487629, 0.05961218476295471, 0.02238387055695057, 0.016796991229057312, 0.10928259789943695, 0.07264000922441483, 0.08393709361553192, 0.025248989462852478, 0.03774847462773323, 0.016226941719651222, 0.024953871965408325, 0.042165856808423996, 0.004525808617472649, 0.014491353183984756, 0.01914571039378643, 0.022841647267341614]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09646990895271301, 0.9035300612449646, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04325238987803459, 0.08177754282951355, 0.8749701380729675, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09995394200086594, 0.025312727317214012, 0.02010803483426571, 0.8546252846717834, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.024889368563890457, 0.00320735527202487, 0.0018421593122184277, 0.022361503913998604, 0.9476996660232544, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10732383280992508, 0.017841672524809837, 0.01955333538353443, 0.04333319514989853, 0.10211498290300369, 0.7098329067230225, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.006426361855119467, 0.00044797800364904106, 0.00014756589371245354, 0.00046936701983213425, 0.0014411886222660542, 0.003859696676954627, 0.98720782995224, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00103545852471143, 0.00019901820633094758, 0.00016020411567296833, 6.937271973583847e-05, 0.00038674141978845, 0.005171590484678745, 0.8964057564735413, 0.0965719074010849, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0012883321614935994, 0.0003233453317079693, 0.00026527189766056836, 0.00025490616098977625, 0.000201298258616589, 0.00010049015691038221, 0.0005700886249542236, 0.000409130152547732, 0.9965871572494507, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0001874157169368118, 1.6196530850720592e-05, 4.2281039895897266e-06, 0.000287588540231809, 1.1125704986625351e-05, 9.805656191019807e-06, 0.0001556719362270087, 7.632971392013133e-05, 0.0034869094379246235, 0.9957647323608398, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015741512179374695, 0.0006393603980541229, 0.00045715278247371316, 0.0009912519017234445, 0.0021140354219824076, 0.0180897768586874, 0.0471203438937664, 0.07010912150144577, 0.06152832508087158, 0.27690398693084717, 0.5063051581382751, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0006166594685055315, 0.0005207078065723181, 4.61151976196561e-05, 0.00146130018401891, 0.0005623754695989192, 4.4476037146523595e-05, 0.00036539664142765105, 0.0002860166132450104, 0.004506127443164587, 0.005816521123051643, 0.0007244544103741646, 0.9850499033927917, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0010478557087481022, 3.062814721488394e-05, 0.00017344979278277606, 5.423119000624865e-05, 6.388417386915535e-05, 1.1261126928729936e-05, 1.7169008060591295e-05, 1.3931307876191568e-05, 0.0020760390907526016, 0.00026926834834739566, 0.00015268517017830163, 0.0036844322457909584, 0.992405116558075, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.010746272280812263, 0.00025354334502480924, 0.0001532292371848598, 0.00029287015786394477, 0.0005376793560571969, 0.004423544742166996, 0.00988433975726366, 0.012843050062656403, 0.012738612480461597, 0.05966852977871895, 0.107729971408844, 0.024745700880885124, 0.07808694988489151, 0.6778956651687622, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00041218000114895403, 2.6218738639727235e-05, 1.7755904991645366e-05, 0.00019181883544661105, 3.2979492061713245e-06, 3.912682132067857e-06, 1.043809788825456e-05, 4.9068494263337925e-06, 0.0005868570297025144, 0.003038151189684868, 3.693124745041132e-05, 0.0007724304450675845, 0.009622604586184025, 0.00016094396414700896, 0.9851114153862, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00010886583186220378, 3.5845487218466587e-06, 5.191657237446634e-06, 3.083837145823054e-05, 1.8408762116450816e-05, 7.954765806061914e-07, 3.3532073757669423e-06, 6.574365215783473e-06, 0.0007270933128893375, 0.0018232465954497457, 7.853259376133792e-06, 0.0003062748000957072, 0.006975048687309027, 3.4850869269575924e-05, 0.0282927006483078, 0.9616552591323853, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.006073986645787954, 8.815194451017305e-05, 4.5837816287530586e-05, 8.250484825111926e-05, 0.0001272043155040592, 0.0009312513284385204, 0.0016699967673048377, 0.001938193803653121, 0.002101697726175189, 0.010131671093404293, 0.016047311946749687, 0.003835388459265232, 0.012242135591804981, 0.10037057846784592, 0.07853133976459503, 0.10997355729341507, 0.655809223651886, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0021862962748855352, 1.1892905604327098e-05, 1.673595943429973e-05, 2.081541060761083e-05, 2.116251016559545e-05, 0.000771492428611964, 0.0008652104297652841, 0.0005560291465371847, 0.0001565588463563472, 0.0019499900517985225, 0.010891195386648178, 0.0003954307467211038, 0.0015144682256504893, 0.06916307657957077, 0.00430810172110796, 0.004035161342471838, 0.4939703941345215, 0.40916600823402405, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [1.979802800633479e-05, 4.972388865098765e-07, 9.16571263331889e-09, 1.3230416584519844e-07, 5.6018219396491986e-08, 1.1903876995233986e-08, 6.153105118755775e-07, 4.0685506519366754e-08, 4.517455181485275e-06, 1.4017791727383155e-05, 1.6456915830076468e-07, 1.086384372683824e-06, 7.44406133890152e-06, 1.005667400022503e-06, 2.75950224022381e-05, 0.00031378038693219423, 6.201125415827846e-06, 8.213370165321976e-06, 0.9995948672294617, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [6.497167760244338e-06, 8.692019264344708e-07, 1.4051378229851252e-06, 3.7704828059759166e-07, 3.3291723866568645e-07, 6.104961869368708e-08, 2.542995503063139e-08, 1.1067748317827864e-07, 3.670873775263317e-05, 5.704808927475824e-07, 6.172335247356386e-07, 0.00015627949323970824, 0.00014737885794602334, 3.010375849044067e-06, 6.130654219305143e-05, 0.0017769918777048588, 1.8545919374446385e-05, 3.426316106924787e-05, 0.923351526260376, 0.07440318167209625, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.003649001009762287, 3.7494322896236554e-05, 1.74708020495018e-05, 2.8449197998270392e-05, 3.312424087198451e-05, 0.00019569217693060637, 0.0002647593501023948, 0.0002469043538440019, 0.00028033158741891384, 0.0013949184212833643, 0.0017910305177792907, 0.00042468024184927344, 0.001462370972149074, 0.009717755019664764, 0.008053947240114212, 0.01320058386772871, 0.06315966695547104, 0.07152792811393738, 0.07523112744092941, 0.024205094203352928, 0.7250776290893555, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.000641089805867523, 3.925701548723737e-06, 1.1607372698563267e-06, 1.537548200758465e-06, 9.072650755115319e-07, 8.05819763627369e-06, 1.3223971109255217e-05, 4.6199284042813815e-06, 3.678836583276279e-05, 4.5651795517187566e-05, 7.577823998872191e-05, 2.274175494676456e-05, 5.6684530136408284e-05, 0.0004737511626444757, 0.0003640766954049468, 0.00042400247184559703, 0.00375733501277864, 0.004363126587122679, 0.007395228371024132, 0.005022629629820585, 0.05575620383024216, 0.9215314984321594, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0005377684719860554, 8.12255109394755e-07, 2.3018603769742185e-06, 1.470038341722102e-06, 1.8703678961173864e-06, 6.02451621034561e-07, 4.12783674619277e-06, 1.6705761254343088e-06, 2.047191856036079e-06, 7.430045661749318e-05, 3.5942291560786543e-06, 7.171521247073542e-06, 3.052820102311671e-05, 1.8927734345197678e-05, 0.0002543879672884941, 0.0001949639990925789, 0.00012626573152374476, 0.00018230534624308348, 0.0037352300714701414, 0.0006727128056809306, 0.0015798343811184168, 0.011177700012922287, 0.9813893437385559, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [8.512814383720979e-05, 4.870869929618493e-07, 1.2458942819648655e-06, 2.0811235401652084e-07, 1.92428601053507e-07, 3.704030859807972e-08, 4.5552428673545364e-07, 5.504545086409962e-08, 1.532036321805208e-06, 5.083304131403565e-06, 2.1400977345820138e-07, 1.459531858927221e-06, 1.4768386790819932e-05, 1.0635022817950812e-06, 2.9461503800121136e-05, 5.7288212701678276e-05, 6.615699021494947e-06, 7.844223546271678e-06, 0.00015063839964568615, 0.00022636978246737272, 7.973578613018617e-05, 0.00013040869089309126, 0.010868428274989128, 0.988331139087677, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0004166325961705297, 2.250373881906853e-06, 5.27054771737312e-06, 2.7549118385650218e-05, 3.963056951761246e-05, 7.514749540860066e-06, 8.10094024927821e-06, 1.1581340913835447e-05, 1.320597675658064e-05, 6.750728789484128e-05, 2.5212739274138585e-05, 1.486069868406048e-05, 7.521701627410948e-05, 0.00011457462824182585, 0.0002574531245045364, 0.0003666863194666803, 0.0006904753972776234, 0.0014518487732857466, 0.0032460209913551807, 0.0006353403441607952, 0.008745575323700905, 0.015190862119197845, 0.01815020479261875, 0.1330331563949585, 0.8174033164978027, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [1.673920633038506e-05, 8.245287830277448e-08, 3.257373748510872e-08, 2.873728810470766e-08, 1.6217634879467369e-07, 1.5275355425004022e-09, 2.3278557748085404e-08, 5.576718198341268e-09, 1.2825742601307866e-07, 1.3429436762635305e-07, 6.160537857624604e-09, 2.5232478151338e-07, 3.2057448606792605e-06, 2.6277179543399143e-08, 7.619933626301645e-07, 7.901945537014399e-06, 1.3940400833689637e-07, 1.955255868324457e-07, 0.00032022525556385517, 8.173647074727342e-07, 1.562996089887747e-06, 3.763153017644072e-06, 0.0007450500270351768, 0.0037152147851884365, 3.934417327400297e-05, 0.9951443076133728, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00023842290102038532, 5.040124960942194e-06, 1.3208604059400386e-06, 8.599836291978136e-06, 5.166078608453972e-06, 1.3470925352976337e-07, 6.710816364829952e-07, 2.699698029573483e-07, 6.345731435430935e-06, 4.417831223690882e-05, 2.7444934858067427e-07, 2.017119368247222e-05, 1.652665923757013e-05, 8.122791541609331e-07, 0.0001077100241673179, 0.0001239744306076318, 3.5075922824034933e-06, 8.505840924044605e-06, 6.980268517509103e-05, 0.0002681369078345597, 2.8792866942239925e-05, 0.00011957906826864928, 0.0001296169066336006, 0.007425930816680193, 0.0007109709549695253, 0.005815016105771065, 0.9848405718803406, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00040899505256675184, 6.095365279179532e-07, 1.9963911199738504e-06, 9.31032991502434e-06, 3.769046315937885e-06, 1.987656105484348e-06, 4.57654266483587e-07, 1.069074301085493e-06, 3.087174036409124e-06, 9.211397809849586e-06, 4.724369318864774e-06, 1.2070032653355156e-06, 5.095620508654974e-06, 1.8047117919195443e-05, 4.857865133089945e-05, 4.169386738794856e-05, 0.00010537513298913836, 0.00022466867812909186, 0.00011528613686095923, 0.0002492894127499312, 0.0013673142530024052, 0.0016190343303605914, 0.0013275155797600746, 0.003491780487820506, 0.06305663287639618, 0.00646906066685915, 0.0951264277100563, 0.8262877464294434, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00031804729951545596, 1.4181842971083825e-06, 3.1855009297032666e-07, 2.7823105028801365e-06, 5.388588397181593e-05, 4.300602540752152e-08, 5.386270913732005e-07, 8.497841008647811e-08, 5.99971485826245e-07, 4.110329882678343e-06, 7.099399113030813e-08, 8.454308044747449e-07, 2.5266151624236954e-06, 2.198386539475905e-07, 6.098252015362959e-06, 2.1130046661710367e-05, 9.411802466274821e-07, 9.95040522866475e-07, 3.041357376787346e-05, 2.976995119752246e-06, 8.453868758806493e-06, 2.6007013730122708e-05, 0.00023055852216202766, 0.0013383383629843593, 0.0002175826084567234, 0.007014401722699404, 0.017924025654792786, 0.0005302676581777632, 0.9722622632980347, 0.0, 0.0, 0.0, 0.0], [1.8505332377571904e-07, 5.565692351439111e-09, 7.262683787701008e-10, 7.745998509278706e-09, 1.507951097323712e-08, 8.22852175286215e-10, 1.7915348093922034e-09, 1.7345647140842857e-09, 1.0962556196147943e-08, 5.0032749498996054e-08, 3.9228162940219136e-09, 1.0013924622853665e-07, 2.333774595797422e-08, 1.6555254944705666e-08, 5.1713197279923406e-08, 1.614605764643784e-07, 9.827901692460728e-08, 2.707449482386437e-07, 5.411987331171986e-06, 2.5083886612264905e-07, 1.1805956319221877e-06, 2.8004205887555145e-06, 4.61038098364952e-06, 0.00022679210815113038, 5.7942423154599965e-05, 0.0004519550711847842, 0.0001197646779473871, 0.00023980921832844615, 0.001874006469734013, 0.9970145225524902, 0.0, 0.0, 0.0], [4.347462163423188e-05, 2.968874923681142e-06, 3.919368793958711e-07, 9.117065360442211e-07, 4.364489711861097e-07, 4.686591115188321e-09, 9.240378062713717e-08, 1.2419349992853768e-08, 7.188964445958845e-08, 6.304543944679608e-07, 6.108888950251412e-09, 1.9987160726486763e-07, 1.4578674836229766e-06, 1.4727343788933922e-08, 1.7957326292616926e-07, 2.6422712835483253e-05, 5.112656609185251e-08, 4.710674161856332e-08, 0.0015761180547997355, 2.198941047026892e-06, 3.3339856031489035e-07, 2.1814876163261943e-06, 0.00016240392869804054, 0.0017777654575183988, 1.1749207260436378e-05, 0.0036160903982818127, 6.199482595548034e-05, 3.5168231988791376e-05, 0.001148868934251368, 0.016085581853985786, 0.9754422307014465, 0.0, 0.0], [0.0011562927393242717, 6.199047675181646e-06, 6.137794912319805e-07, 3.102660230069887e-06, 8.228408319155278e-07, 4.3660193682626414e-08, 9.36325577072239e-08, 2.7735543639550997e-08, 5.109529865876539e-07, 1.2355309308986762e-06, 3.8253045175906664e-08, 1.127230689235148e-06, 1.9008897425010218e-06, 8.714057031511402e-08, 5.398183020588476e-06, 7.393444320769049e-06, 3.128881189695676e-07, 3.0146313179102435e-07, 0.00013595378550235182, 1.3343835234991275e-05, 2.39964992942987e-06, 3.871273293043487e-05, 0.0008219636511057615, 0.0006131280679255724, 2.151841727027204e-05, 0.004140730947256088, 0.0008760862983763218, 0.00011684626952046528, 0.0021590692922472954, 0.027525268495082855, 0.04434971883893013, 0.9179997444152832, 0.0], [0.003194290678948164, 4.681676728068851e-05, 3.49913134414237e-05, 2.7043113732361235e-05, 1.4000337614561431e-05, 1.7085165382013656e-05, 8.051893928495701e-06, 3.909831775672501e-06, 1.0716959877754562e-05, 1.793081537471153e-05, 1.4207512322172988e-05, 2.691133522603195e-05, 2.6411998987896368e-05, 3.1100229534786195e-05, 1.1198581887583714e-05, 5.485505607794039e-05, 0.00011474490747787058, 0.00010493675654288381, 0.00028063071658834815, 0.00020839076023548841, 0.0009329256135970354, 0.0018990016542375088, 0.0011267587542533875, 0.0019257268868386745, 0.0039171562530100346, 0.007499460596591234, 0.010695938020944595, 0.023199880495667458, 0.03039817325770855, 0.12729351222515106, 0.04325327277183533, 0.1766805499792099, 0.5669293999671936]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.25308629870414734, 0.746913731098175, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.30671578645706177, 0.32906386256217957, 0.3642203211784363, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07416975498199463, 0.1618966907262802, 0.054325949400663376, 0.7096075415611267, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1668822318315506, 0.039017826318740845, 0.03822460398077965, 0.21398353576660156, 0.5418918132781982, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19483636319637299, 0.2111925631761551, 0.05150565132498741, 0.0870383232831955, 0.2299954891204834, 0.22543159127235413, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13900135457515717, 0.02974863536655903, 0.03860695660114288, 0.05133272334933281, 0.19284246861934662, 0.08012373745441437, 0.4683440625667572, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08969778567552567, 0.04080599546432495, 0.03473307564854622, 0.08414525538682938, 0.09911047667264938, 0.07059449702501297, 0.1361657828092575, 0.4447471499443054, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05986519902944565, 0.020190449431538582, 0.018785323947668076, 0.1058472990989685, 0.057948824018239975, 0.027517711743712425, 0.0566631518304348, 0.08226760476827621, 0.5709145069122314, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.010379260405898094, 0.0046982537023723125, 0.004143984522670507, 0.0072914063930511475, 0.006256693042814732, 0.0033180469181388617, 0.009175012819468975, 0.018754413351416588, 0.03393147885799408, 0.9020513892173767, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07270114123821259, 0.0477648489177227, 0.010500345379114151, 0.0161594245582819, 0.046178195625543594, 0.05249456688761711, 0.051845718175172806, 0.21189512312412262, 0.035158202052116394, 0.17267179489135742, 0.2826306223869324, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015126891434192657, 0.004731189925223589, 0.0023226316552609205, 0.0065757399424910545, 0.01836245507001877, 0.003339666873216629, 0.008784224279224873, 0.007409745827317238, 0.006289808079600334, 0.07638943940401077, 0.01200348511338234, 0.838664710521698, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.03233193978667259, 0.010628965683281422, 0.0026151640340685844, 0.0011762931244447827, 0.0030932866502553225, 0.0015055524418130517, 0.007079718168824911, 0.00283446884714067, 0.005003884434700012, 0.012032588012516499, 0.0039877742528915405, 0.051872409880161285, 0.8658380508422852, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05045590549707413, 0.02653883770108223, 0.005783271975815296, 0.00808729324489832, 0.021799379959702492, 0.025670334696769714, 0.022779036313295364, 0.09139905869960785, 0.01501737255603075, 0.07093921303749084, 0.12448635697364807, 0.1266833245754242, 0.0610213503241539, 0.34933918714523315, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05474122613668442, 0.05296846479177475, 0.00398812722414732, 0.012351620942354202, 0.00441562058404088, 0.0035962786059826612, 0.011385707184672356, 0.009828065522015095, 0.014749741181731224, 0.07078825682401657, 0.011209187097847462, 0.05164055898785591, 0.1973438560962677, 0.027129262685775757, 0.4738638997077942, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.007163074798882008, 0.004963848739862442, 0.0027692278381437063, 0.0019424431957304478, 0.010544748045504093, 0.0014144877204671502, 0.003663665847852826, 0.003149948548525572, 0.005481299012899399, 0.021614043042063713, 0.003922200761735439, 0.07935082912445068, 0.2247791737318039, 0.009166955016553402, 0.027402665466070175, 0.5926714539527893, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.034611064940690994, 0.015296351164579391, 0.003095126012340188, 0.0038990958128124475, 0.009989401325583458, 0.01144686620682478, 0.009189358912408352, 0.03356517106294632, 0.005761212203651667, 0.025581011548638344, 0.043846867978572845, 0.04521361365914345, 0.02055339701473713, 0.1196233481168747, 0.1567324846982956, 0.13173452019691467, 0.32986119389533997, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.017253872007131577, 0.004397525917738676, 0.004968690220266581, 0.007064312230795622, 0.006634894758462906, 0.006910949479788542, 0.011386900208890438, 0.016178512945771217, 0.020317763090133667, 0.020396802574396133, 0.02342209964990616, 0.01890912838280201, 0.04024204611778259, 0.06200675293803215, 0.116453617811203, 0.07975102961063385, 0.16864468157291412, 0.37506037950515747, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.002459116280078888, 0.00024919764837250113, 3.400466084713116e-05, 0.0002113294176524505, 0.00020066798606421798, 0.0001258924457943067, 0.00036506837932392955, 0.00037359801353886724, 0.00013798549480270594, 0.0004702212172560394, 0.00027151827816851437, 0.003417074913159013, 0.0006049809162504971, 0.0005454609636217356, 0.0008937679813243449, 0.0015852133510634303, 0.0012150758411735296, 0.00282126828096807, 0.9840186238288879, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00015282434469554573, 0.038923632353544235, 0.00047151994658634067, 0.0003731817996595055, 3.271793684689328e-05, 1.063560375769157e-05, 3.8245766518230084e-06, 1.3244694855529815e-05, 6.418924022000283e-05, 2.7344345653546043e-05, 2.8319956982159056e-05, 0.0022295680828392506, 0.0013866383815184236, 7.163518603192642e-05, 0.0001402555499225855, 0.00824644137173891, 0.00019678636454045773, 0.00020546668383758515, 0.8976381421089172, 0.049783725291490555, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02645810693502426, 0.010410889983177185, 0.0021332555916160345, 0.0024560203310102224, 0.005542597267776728, 0.005954570136964321, 0.0041958331130445, 0.013451533392071724, 0.0023519538808614016, 0.009471958503127098, 0.015433433465659618, 0.014804964885115623, 0.006865760777145624, 0.037873148918151855, 0.047449350357055664, 0.045472823083400726, 0.10199166089296341, 0.08666391670703888, 0.06470184028148651, 0.10160762816667557, 0.3947088122367859, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.021157892420887947, 0.0035484335385262966, 0.001744111767038703, 0.0035093079786747694, 0.004201894626021385, 0.00257777888327837, 0.004015856888145208, 0.004598999861627817, 0.003955156076699495, 0.006455833092331886, 0.00590789457783103, 0.0037331636995077133, 0.009013411588966846, 0.013927980326116085, 0.029895801097154617, 0.02008231170475483, 0.03804440051317215, 0.06883092224597931, 0.0815289318561554, 0.0383402481675148, 0.15160253643989563, 0.48332712054252625, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0014211690286174417, 0.00040690117748454213, 0.0002714421134442091, 0.0014869242440909147, 0.0006475155241787434, 0.0002567152841947973, 0.00027297638007439673, 0.0005059852264821529, 0.00017537492385599762, 0.0012059591244906187, 0.0005902894772589207, 0.00043073901906609535, 0.00036999606527388096, 0.0013322837185114622, 0.000759427435696125, 0.002518307650461793, 0.003599978983402252, 0.004756716545671225, 0.011892932467162609, 0.0034102171193808317, 0.013851108960807323, 0.05907328426837921, 0.8907637596130371, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00505886459723115, 0.0008823543903417885, 0.0015358274104073644, 0.0016909865662455559, 0.0013438736787065864, 0.0004949708236381412, 0.0006572254933416843, 0.000552159093786031, 0.0005812101298943162, 0.002054560463875532, 0.0008380875224247575, 0.001006747712381184, 0.0011300853220745921, 0.0016139923827722669, 0.005022874567657709, 0.014576874673366547, 0.0037143337540328503, 0.004681476857513189, 0.007913822308182716, 0.006778388284146786, 0.012291365303099155, 0.027733024209737778, 0.10214646905660629, 0.7957004308700562, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00400469871237874, 0.0004576975479722023, 0.0015645211096853018, 0.0031372052617371082, 0.001993746031075716, 0.00037892640102654696, 0.00049577810568735, 0.000569801835808903, 0.001005206722766161, 0.0023432248272001743, 0.0005772155127488077, 0.0016756145050749183, 0.003078736364841461, 0.0012285250704735518, 0.005278328433632851, 0.00632805610075593, 0.003151731565594673, 0.0053193108178675175, 0.0213831327855587, 0.006147712003439665, 0.012332223355770111, 0.07402164489030838, 0.1211402490735054, 0.561111569404602, 0.16127507388591766, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0011286542285233736, 4.9889109504874796e-05, 4.129114677198231e-05, 9.24961204873398e-05, 5.513622454600409e-05, 3.8617254176642746e-05, 5.5920278100529686e-05, 0.00014896267384756356, 5.509220500243828e-06, 6.817427492933348e-05, 3.6204455682309344e-05, 4.623961285687983e-05, 0.001065501943230629, 5.91987554798834e-05, 7.787253707647324e-05, 0.00038421223871409893, 0.00011692437692545354, 0.0002242824120912701, 0.000508398690726608, 0.0001562813704367727, 0.00032331488910131156, 0.0007086826371960342, 0.0009207415860146284, 0.0008559715934097767, 0.004414450377225876, 0.9884170889854431, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0024299705401062965, 0.003659637412056327, 0.0006344981375150383, 0.0006202656077221036, 0.00298767676576972, 0.00034763681469485164, 0.0005059166578575969, 0.00041149763274006546, 0.00021321582607924938, 0.0006509345257654786, 0.0003955696302000433, 0.16986453533172607, 0.002188899554312229, 0.0006812380161136389, 0.000868376693688333, 0.008905136026442051, 0.0014638527063652873, 0.0013394575798884034, 0.006495129782706499, 0.007638717070221901, 0.004619399085640907, 0.003161790780723095, 0.014200896956026554, 0.046396613121032715, 0.02278606779873371, 0.21939247846603394, 0.4771406054496765, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015610319562256336, 0.0009648068808019161, 0.0010281825670972466, 0.007148513570427895, 0.005316650029271841, 0.0005432680482044816, 0.0007839886820875108, 0.0004708298947662115, 0.0022980361245572567, 0.005463092587888241, 0.0005483070272020996, 0.0008661497267894447, 0.0009035550756379962, 0.0009251966839656234, 0.008375934325158596, 0.0014478107914328575, 0.0020614899694919586, 0.003693382255733013, 0.017008859664201736, 0.006138675846159458, 0.007041892036795616, 0.04128709062933922, 0.06922487169504166, 0.26617181301116943, 0.05024212971329689, 0.11445137113332748, 0.13764896988868713, 0.23233483731746674, 0.0, 0.0, 0.0, 0.0, 0.0], [0.021895311772823334, 0.0013514558086171746, 0.0013727976474910975, 0.005157721694558859, 0.01093280129134655, 0.00044522207463160157, 0.0016178301302716136, 0.0005713265272788703, 0.0007700274582020938, 0.003093535080552101, 0.0003616345056798309, 0.0009329011081717908, 0.001903240685351193, 0.0005254389252513647, 0.0007036188035272062, 0.0015780640533193946, 0.0010200472315773368, 0.0009346723672933877, 0.005167578347027302, 0.0007855422445572913, 0.0028796829283237457, 0.006337142549455166, 0.04951414093375206, 0.021886898204684258, 0.03004683181643486, 0.06941598653793335, 0.02995387092232704, 0.061116255819797516, 0.6677284240722656, 0.0, 0.0, 0.0, 0.0], [0.0002946656895801425, 8.08511977083981e-05, 0.00011992631334578618, 0.00017154013039544225, 0.0003899005532730371, 8.630252705188468e-05, 5.253727795206942e-05, 6.616376049350947e-05, 0.0001137837243732065, 6.200082134455442e-05, 0.00011238027218496427, 0.0005867315921932459, 9.785001748241484e-05, 0.00021334877237677574, 0.00040444283513352275, 0.0004726238548755646, 0.0004730736545752734, 0.0005199519800953567, 0.0008571160142309964, 0.00022720196284353733, 0.0015976768918335438, 0.0014780627097934484, 0.0016501408535987139, 0.009339644573628902, 0.01279268879443407, 0.012606951408088207, 0.021659579128026962, 0.02886745147407055, 0.07784783840179443, 0.8267576694488525, 0.0, 0.0, 0.0], [0.0006396545795723796, 0.00019920240447390825, 0.0003378460241947323, 0.00032286165514960885, 0.00038839809712953866, 2.2511459974339232e-05, 0.00011559655831661075, 1.968370270333253e-05, 8.749768312554806e-05, 0.00016675743972882628, 2.1353174815885723e-05, 0.00018257695774082094, 0.002879622159525752, 3.2708376238588244e-05, 6.355983350658789e-05, 0.00090785016072914, 6.31420552963391e-05, 4.9545076763024554e-05, 0.005010190419852734, 0.00021555765124503523, 0.00018923325114883482, 0.00023579830303788185, 0.0021191714331507683, 0.0034752420615404844, 0.0004957506898790598, 0.012673038989305496, 0.0026274591218680143, 0.0011926377192139626, 0.028011513873934746, 0.1421458125114441, 0.7951081991195679, 0.0, 0.0], [0.005492142401635647, 0.0014179605059325695, 6.310038588708267e-05, 0.0009282368700951338, 0.0004675872914958745, 9.701211820356548e-05, 5.678690649801865e-05, 6.824053707532585e-05, 4.054806777276099e-05, 0.00021011428907513618, 6.534699787152931e-05, 0.00011957778770010918, 9.082323231268674e-05, 9.077344293473288e-05, 0.00013827496150042862, 0.00041228361078538, 0.00016900571063160896, 0.00013570708688348532, 0.002428320934996009, 0.00020654806576203555, 0.0004959595971740782, 0.00016561975644435734, 0.001825400977395475, 0.001680815708823502, 0.001018735347315669, 0.01507147029042244, 0.003408371703699231, 0.003384194802492857, 0.01886550709605217, 0.010037437081336975, 0.008259555324912071, 0.923088550567627, 0.0], [0.009386110119521618, 0.0017380556091666222, 0.0018091682577505708, 0.0014350252458825707, 0.0016812816029414535, 0.001979130320250988, 0.0014143437147140503, 0.0021127830259501934, 0.0012016963446512818, 0.0010467646643519402, 0.001253248774446547, 0.0012125695357099175, 0.0006952421390451491, 0.0016850176034495234, 0.001290497719310224, 0.0014949428150430322, 0.003052463522180915, 0.006176403723657131, 0.003993155900388956, 0.002994521288201213, 0.008827597834169865, 0.009542387910187244, 0.010477354750037193, 0.0214514322578907, 0.062213558703660965, 0.026861043646931648, 0.033906422555446625, 0.11987609416246414, 0.05027468875050545, 0.026796849444508553, 0.031165700405836105, 0.10728635638952255, 0.4436681270599365]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10520488768815994, 0.8947951793670654, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.03907204046845436, 0.0020172216463834047, 0.9589106440544128, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015578641556203365, 0.0008392541203647852, 0.0006979768513701856, 0.9828841090202332, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.008856471627950668, 9.934287845680956e-06, 1.1174832252436318e-05, 0.000302469969028607, 0.9908198714256287, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.35445019602775574, 0.030589278787374496, 0.059889569878578186, 0.022903528064489365, 0.0474759079515934, 0.4846915602684021, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04476895555853844, 0.001566466293297708, 0.00037738497485406697, 0.0002507324970792979, 0.00040889676893129945, 0.0002606561465654522, 0.9523669481277466, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09058346599340439, 0.0011470747413113713, 0.006027346942573786, 0.000546847702935338, 0.0017094232607632875, 0.00378508516587317, 0.002684543142095208, 0.8935161828994751, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0060107228346168995, 2.4289069187943824e-05, 0.00031266745645552874, 1.8682934751268476e-05, 0.0002979248820338398, 9.904525541060138e-06, 6.619561190746026e-06, 7.912206569926639e-07, 0.9933184385299683, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.004546814598143101, 0.0002001934335567057, 0.00029920213273726404, 0.001336393179371953, 0.0003267655265517533, 8.741358215047512e-07, 1.7415688489563763e-05, 2.783414174700738e-07, 0.00029665243346244097, 0.9929754734039307, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12262558937072754, 0.02392657659947872, 0.03827648609876633, 0.016171958297491074, 0.030366230756044388, 0.29375940561294556, 0.10661352425813675, 0.0705994963645935, 0.04190007597208023, 0.017833102494478226, 0.23792758584022522, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0008121057762764394, 0.00011546660243766382, 0.00013500906061381102, 5.5392923968611285e-05, 0.0001317415590165183, 1.0027196140072192e-06, 1.3799102589473478e-06, 1.6001629887796298e-07, 1.8972989437315846e-06, 5.419660737970844e-07, 4.753100597554294e-07, 0.9987448453903198, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0013333020033314824, 0.00010533027671044692, 0.0007669601473025978, 0.0009021890582516789, 3.5708292216440896e-06, 8.120475740724942e-07, 1.4038076301403635e-07, 9.5612755046659e-08, 1.6865038787727826e-06, 2.9789011023240164e-05, 4.3813363959088747e-07, 1.1965776138822548e-06, 0.9968544840812683, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08159798383712769, 0.021388858556747437, 0.028877155855298042, 0.012675793841481209, 0.02493158169090748, 0.229964941740036, 0.10401230305433273, 0.06472097337245941, 0.03460092470049858, 0.015198714099824429, 0.18564164638519287, 0.020528046414256096, 0.005003852304071188, 0.17085722088813782, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0013488862896338105, 8.92121170181781e-05, 0.000685996375977993, 0.0006963219493627548, 2.1248622942948714e-05, 3.7633941474268795e-07, 6.2267383782455e-07, 4.3072165567537013e-07, 7.091880888765445e-06, 0.0013103694655001163, 1.611375211041377e-07, 9.54876213654643e-07, 0.00016453623538836837, 1.15319252813606e-07, 0.9956737160682678, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0010867505334317684, 0.000407249026466161, 0.00010700321581680328, 0.000678271462675184, 0.00011028484004782513, 1.318426257057581e-07, 2.660271150034532e-07, 4.6195768277357274e-07, 8.409714610024821e-06, 0.0007784898625686765, 5.109449574547398e-08, 8.792079029262823e-07, 4.934294702252373e-05, 3.4893233191723994e-08, 0.000114411988761276, 0.9966580867767334, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05868088826537132, 0.0187857486307621, 0.024067312479019165, 0.011013144627213478, 0.021456381306052208, 0.1911056488752365, 0.09902721643447876, 0.06065867841243744, 0.03016272373497486, 0.014801910147070885, 0.15370330214500427, 0.016615159809589386, 0.004385382868349552, 0.14085331559181213, 0.015014998614788055, 0.0053902058862149715, 0.13427793979644775, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07323037832975388, 0.02837456949055195, 0.02105824276804924, 0.007611881475895643, 0.01005358062684536, 0.06553948670625687, 0.14559759199619293, 0.08620844036340714, 0.01024126447737217, 0.007425738964229822, 0.045262690633535385, 0.006621456239372492, 0.001162177650257945, 0.03923853859305382, 0.008256880566477776, 0.0030656892340630293, 0.036122631281614304, 0.40492871403694153, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [7.94449370005168e-05, 1.8382123698756914e-06, 3.7134324060161816e-08, 1.5575307770632207e-05, 3.4723757380561437e-06, 1.3165096790501707e-09, 2.703870904952055e-08, 1.7186652101486288e-08, 9.2322629541286e-08, 1.350637921859743e-05, 4.4787887287789374e-10, 2.041673052843862e-09, 5.5420287026208825e-08, 3.0638666603799436e-10, 4.217719435928302e-07, 7.065880708978511e-06, 2.424439260462208e-10, 1.4397358860662735e-09, 0.9998784065246582, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.002934493590146303, 0.0005883841076865792, 0.004250307101756334, 0.0040284618735313416, 0.00011592944065341726, 4.3316217670508195e-06, 6.773899713152787e-06, 7.076467591105029e-05, 0.0002495265216566622, 0.0003159338375553489, 2.08243977795064e-06, 4.4092124880990013e-05, 0.00030446931486949325, 1.4702561657031765e-06, 0.0001976135972654447, 0.0013599261874333024, 1.2478651569836074e-06, 6.875188319099834e-06, 0.00019261146371718496, 0.9853246808052063, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04056532680988312, 0.017898326739668846, 0.020600972697138786, 0.009821100160479546, 0.01788281463086605, 0.15653997659683228, 0.09208228439092636, 0.05628826469182968, 0.026661338284611702, 0.014471272937953472, 0.12537629902362823, 0.013986770063638687, 0.00392792746424675, 0.11453618854284286, 0.013395379297435284, 0.004846664611250162, 0.10880246758460999, 0.049533966928720474, 0.005186409689486027, 0.004420871380716562, 0.10317535698413849, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02453630231320858, 0.0031243355479091406, 0.0009699289221316576, 4.6315435611177236e-05, 0.0001343475450994447, 0.0006453939131461084, 0.011668838560581207, 0.00032713444670662284, 0.0004573040350805968, 2.3976079319254495e-05, 0.0003445417678449303, 4.996419738745317e-05, 2.5757384719327092e-05, 0.0002821775560732931, 6.525318895000964e-06, 4.981805432180408e-06, 0.00023554006475023925, 0.0002664509229362011, 2.531113386794459e-05, 2.9761044061160646e-05, 0.00018809252651408315, 0.9566071033477783, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.002076001837849617, 0.00014591813669539988, 5.28989803569857e-05, 4.218077810946852e-05, 4.7387235099449754e-05, 4.14233511492057e-07, 0.00012964299821760505, 2.4251021386589855e-05, 1.8034043023362756e-06, 1.4839295090496307e-06, 1.4490007060885546e-07, 4.199936029181117e-06, 4.1068410894240515e-08, 1.0260500005188078e-07, 8.285463763968437e-07, 5.781907930213492e-07, 7.69515864362802e-08, 6.669622365507166e-08, 6.47262713755481e-05, 1.5700916264904663e-06, 5.6418304694716426e-08, 4.403035745781381e-06, 0.997401237487793, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0005159107386134565, 0.00011814631579909474, 3.2705611374694854e-05, 0.00021188720711506903, 4.563520269584842e-05, 2.924884086041857e-07, 3.0579394660890102e-06, 7.607361141026558e-08, 0.0001375367137370631, 0.0005283543141558766, 1.1576013747571778e-07, 4.028577222925378e-06, 2.079294745271909e-06, 8.805408668877135e-08, 0.00010051687422674149, 4.43161916336976e-05, 6.963698950812613e-08, 9.373040654736542e-08, 3.337768794153817e-05, 1.5888888810877688e-05, 5.6106753021367695e-08, 7.05264184830412e-08, 9.389270417159423e-06, 0.9981963038444519, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.06915225088596344, 0.014032105915248394, 0.005599097348749638, 0.0012576853623613715, 0.0008468001033179462, 0.009601173922419548, 0.008965768851339817, 0.052476901561021805, 0.00025028391974046826, 0.0009063858306035399, 0.005419905763119459, 0.000730921165086329, 0.0001918794005177915, 0.004396580625325441, 0.005964720156043768, 0.00035585410660132766, 0.0038770169485360384, 0.009727226570248604, 0.0011023489059880376, 0.0003644288226496428, 0.0034139587078243494, 0.0030523554887622595, 0.005698245484381914, 0.00029557300149463117, 0.7923205494880676, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0010461547644808888, 1.9557141058612615e-05, 1.7358959212288028e-06, 3.1030376703711227e-05, 2.73079531325493e-05, 1.0180698062356441e-08, 4.815767624677392e-07, 1.749424853869641e-07, 7.31240987761339e-08, 7.983068712746899e-07, 3.4643101720632785e-09, 4.9067175496020354e-06, 1.0718932230702194e-07, 2.352543715389288e-09, 2.3834076046114205e-07, 1.3731333865507622e-06, 1.828867168818249e-09, 6.247609984910696e-09, 9.208339179167524e-06, 1.6384256014134735e-06, 1.2523899695082719e-09, 8.271274221094416e-10, 8.456770046905149e-06, 3.118676431768108e-06, 9.241420961814129e-09, 0.9988435506820679, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0010544945253059268, 0.00021373217168729752, 0.00028157219639979303, 0.0011373378802090883, 0.00014783968799747527, 2.00685326490202e-06, 4.831355909118429e-06, 3.48233857039304e-06, 5.100344878883334e-06, 0.000124204860185273, 9.301529075855797e-07, 1.0582622962829191e-05, 1.1688286576827522e-05, 7.077518944242911e-07, 0.003411600599065423, 0.00019905276712961495, 5.995618153065152e-07, 4.643046622732072e-07, 1.9585138943511993e-05, 0.00011087791790487245, 4.7500583377768635e-07, 3.2808881655910227e-07, 4.535358584689675e-06, 5.1617142162285745e-05, 3.247476740853017e-07, 1.3071385183138773e-05, 0.9931889772415161, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02420075610280037, 0.002820128807798028, 0.004319296684116125, 0.0009108306840062141, 0.0017753936117514968, 0.003809914458543062, 0.0029533228371292353, 0.01671477034687996, 0.001393173704855144, 0.005003183614462614, 0.002276088809594512, 0.0005876408540643752, 0.0009901623707264662, 0.0018066932680085301, 0.0031312420032918453, 0.00023389369016513228, 0.0016082713846117258, 0.00991030503064394, 0.002853952581062913, 0.00020495128410402685, 0.0013712650397792459, 0.00691900635138154, 0.0010336573468521237, 0.00014528384781442583, 0.017489798367023468, 0.00046553125139325857, 0.00030240853084251285, 0.8847692012786865, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0005256092990748584, 3.528650950102019e-06, 1.703768475636025e-06, 9.408179903402925e-05, 0.6647976040840149, 1.6531491553450905e-07, 9.511112466498162e-07, 8.886883620107255e-07, 1.701671317277942e-05, 3.7421868910314515e-05, 6.832266308265389e-08, 9.810898518480826e-06, 7.548737102069936e-08, 4.9632518539510784e-08, 2.0393888462422183e-06, 7.686546268814709e-06, 4.066851388984105e-08, 2.6744315206883584e-08, 3.343187927384861e-05, 4.341078749803273e-07, 3.007988169656528e-08, 3.206056931048806e-08, 4.900574822386261e-06, 3.0609394343628082e-06, 1.4603195985785078e-08, 2.2145539332996123e-05, 9.21115315577481e-06, 3.164272044386962e-08, 0.33442792296409607, 0.0, 0.0, 0.0, 0.0], [0.004677004646509886, 0.0002802134840749204, 8.346093818545341e-05, 3.157015271426644e-06, 4.065350367454812e-07, 1.334351367177078e-07, 2.6423560484545305e-05, 1.14399881567806e-07, 2.787234905099467e-07, 3.7496099594136467e-06, 4.652897445112103e-08, 1.7213982346220291e-06, 1.8165600579322927e-07, 3.1484699292150253e-08, 1.8334249318741058e-07, 5.2147463236451586e-08, 2.5215808108214333e-08, 3.078440968806717e-08, 7.142029545548212e-08, 1.6640171907056356e-08, 1.7768222448921733e-08, 8.634525272555038e-08, 2.80096719507128e-06, 8.454135809188301e-07, 3.098357126418705e-08, 2.927866091795295e-07, 5.67622805647261e-07, 5.0554116448608966e-09, 2.9141034474378102e-08, 0.9949179887771606, 0.0, 0.0, 0.0], [5.636792411678471e-05, 0.00020023711840622127, 1.6319079350068932e-06, 1.5089327689565835e-06, 1.7779733752831817e-05, 5.95644145118257e-10, 6.940369075891795e-06, 5.034019867622419e-09, 2.1682852491267113e-07, 1.7164256860269234e-06, 1.9326265587871205e-10, 1.8911425314627195e-08, 9.538840117784275e-08, 1.2946883842790413e-10, 8.07715050399338e-09, 4.386615546536632e-05, 9.768683778554887e-11, 5.256080015669795e-10, 6.93216934450902e-05, 7.505880716962565e-08, 7.069290985928234e-11, 3.988046870517792e-08, 1.0610276149236597e-05, 2.2120707399153616e-06, 8.688633701403603e-10, 9.928487088473048e-06, 5.390084378831261e-09, 1.1354458345769203e-10, 3.060037442992325e-06, 3.8945552205404965e-07, 0.9995738863945007, 0.0, 0.0], [0.00035569758620113134, 1.9489458281896077e-05, 9.160975423583295e-06, 1.3085252248856705e-05, 2.3808875994291157e-05, 2.97190183573548e-08, 4.1605944716138765e-05, 1.7756508441379992e-07, 6.896241870890663e-07, 1.050624064191652e-06, 1.2230765733534099e-08, 4.290548076824052e-06, 3.295830310889869e-07, 9.21108966878137e-09, 1.7466413737565745e-06, 4.68278949483647e-06, 7.4427397578347154e-09, 9.673819079125678e-09, 3.9665097574470565e-05, 7.324898660954204e-07, 5.5436903956262995e-09, 1.0396706073834139e-07, 1.4401575754163787e-05, 8.873340448189992e-06, 5.98941607421466e-09, 1.5627783795935102e-05, 2.9811219519615406e-06, 6.874342872720263e-09, 4.854650796914939e-06, 7.0722649070376065e-06, 6.210685569385532e-06, 0.999423623085022, 0.0], [0.05455271154642105, 0.01642906479537487, 0.01600833795964718, 0.006986881140619516, 0.02488415502011776, 0.07376932352781296, 0.07933780550956726, 0.05594552680850029, 0.02726820856332779, 0.013368779793381691, 0.05590308830142021, 0.007223027292639017, 0.005426580552011728, 0.05012328550219536, 0.00827906746417284, 0.0030609637033194304, 0.046822965145111084, 0.04751850664615631, 0.0028449445962905884, 0.007037974428385496, 0.042827170342206955, 0.0513819120824337, 0.02728237397968769, 0.026375338435173035, 0.026159211993217468, 0.011997989378869534, 0.006220121867954731, 0.03648780658841133, 0.011124704033136368, 0.005218563135713339, 0.0025592022575438023, 0.0026040268130600452, 0.14697040617465973]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9192656874656677, 0.08073433488607407, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.45437392592430115, 0.4265042543411255, 0.11912186443805695, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5267521739006042, 0.22321631014347076, 0.12420626729726791, 0.12582528591156006, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3903489112854004, 0.17099407315254211, 0.06872710585594177, 0.1837182343006134, 0.1862117052078247, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19744226336479187, 0.25436851382255554, 0.14488449692726135, 0.18925853073596954, 0.20821937918663025, 0.00582678010687232, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19473573565483093, 0.13241828978061676, 0.14391927421092987, 0.15120737254619598, 0.3002447485923767, 0.02058463543653488, 0.056889984756708145, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1269666850566864, 0.21597377955913544, 0.16089318692684174, 0.1969882994890213, 0.1884462684392929, 0.013544929213821888, 0.07520709931850433, 0.021979698911309242, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2912123203277588, 0.05954226851463318, 0.11079408973455429, 0.09717857837677002, 0.22345606982707977, 0.016262585297226906, 0.08719441294670105, 0.024762287735939026, 0.08959739655256271, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.22981953620910645, 0.038026757538318634, 0.1601383537054062, 0.11780407279729843, 0.1618119478225708, 0.03552818298339844, 0.07561112195253372, 0.020756039768457413, 0.10670627653598785, 0.05379769951105118, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13314427435398102, 0.18858656287193298, 0.10808095335960388, 0.15994971990585327, 0.1773921251296997, 0.003598014824092388, 0.0250620748847723, 0.01286292728036642, 0.0745309367775917, 0.11275240778923035, 0.004039982799440622, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1674640029668808, 0.1435522586107254, 0.06118571758270264, 0.17358075082302094, 0.10113830864429474, 0.01755347289144993, 0.03239530324935913, 0.007995500229299068, 0.05453447997570038, 0.07883201539516449, 0.017258215695619583, 0.14450997114181519, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.26996830105781555, 0.13029062747955322, 0.06605685502290726, 0.061561424285173416, 0.08437130600214005, 0.01984662190079689, 0.01631278730928898, 0.012965923175215721, 0.03231464698910713, 0.05254792049527168, 0.01986861415207386, 0.13695642352104187, 0.09693864732980728, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09820353239774704, 0.13618646562099457, 0.0800577849149704, 0.12152864784002304, 0.1357937604188919, 0.0024323570542037487, 0.01778949797153473, 0.009086488746106625, 0.05637527257204056, 0.08521638810634613, 0.0027533371467143297, 0.1520378440618515, 0.0994671955704689, 0.00307140382938087, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1701289564371109, 0.08885712921619415, 0.09089689701795578, 0.08085790276527405, 0.11427336931228638, 0.010901479050517082, 0.014788412488996983, 0.005497370846569538, 0.051512569189071655, 0.05330684036016464, 0.010595815256237984, 0.09442280977964401, 0.13098116219043732, 0.011153682135045528, 0.07182558625936508, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10365856438875198, 0.03588540852069855, 0.08803567290306091, 0.06087937951087952, 0.08348330855369568, 0.03080574795603752, 0.029615936800837517, 0.010918705724179745, 0.04049992188811302, 0.04288605973124504, 0.03219296410679817, 0.15008129179477692, 0.12658776342868805, 0.0349021814763546, 0.05825432389974594, 0.07131282985210419, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07047097384929657, 0.0960099846124649, 0.056215789169073105, 0.08699887990951538, 0.09934122860431671, 0.0016298787668347359, 0.01240861602127552, 0.006380431819707155, 0.041014038026332855, 0.06348712742328644, 0.0018482195446267724, 0.11387647688388824, 0.07328979671001434, 0.0020712940022349358, 0.07246166467666626, 0.20012779533863068, 0.002367778215557337, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05659440532326698, 0.08906208723783493, 0.04703241214156151, 0.09457876533269882, 0.08793889731168747, 0.001972166122868657, 0.017767680808901787, 0.005990432109683752, 0.06014860421419144, 0.08834964036941528, 0.0022637685760855675, 0.1046152412891388, 0.06079366058111191, 0.00255662202835083, 0.07818624377250671, 0.19496025145053864, 0.0029299911111593246, 0.004259058274328709, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1597062051296234, 0.026096442714333534, 0.04515180364251137, 0.0349758081138134, 0.08030673116445541, 0.05469036474823952, 0.04078545421361923, 0.02651304565370083, 0.02725193277001381, 0.03919816389679909, 0.05351434648036957, 0.0484292209148407, 0.0626426488161087, 0.05558208376169205, 0.05559664964675903, 0.04792959615588188, 0.05676326900720596, 0.04892301186919212, 0.035943206399679184, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08663122355937958, 0.09557436406612396, 0.036738116294145584, 0.06946705281734467, 0.05681459605693817, 0.00819021463394165, 0.022195832803845406, 0.0076428246684372425, 0.03146827220916748, 0.07555203884840012, 0.0089652631431818, 0.05263480916619301, 0.04491106793284416, 0.009935053065419197, 0.04481377825140953, 0.15198466181755066, 0.0109334047883749, 0.007993670180439949, 0.16402564942836761, 0.01352810300886631, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04783454164862633, 0.06669356673955917, 0.03798917308449745, 0.06040368974208832, 0.06899043172597885, 0.0010267288889735937, 0.008172674104571342, 0.004158170893788338, 0.028268268331885338, 0.044072382152080536, 0.0011586467735469341, 0.0783291831612587, 0.05082900822162628, 0.0013004952343180776, 0.049042947590351105, 0.14145004749298096, 0.0014934344217181206, 0.0025566834956407547, 0.17534081637859344, 0.12910208106040955, 0.0017870558658614755, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04114089906215668, 0.04766072705388069, 0.03586998209357262, 0.06666103005409241, 0.07969274371862411, 0.00236507598310709, 0.02034018747508526, 0.004854495171457529, 0.0643620491027832, 0.06737152487039566, 0.0027073686942458153, 0.05596053972840309, 0.05648668855428696, 0.0030767249409109354, 0.05354386568069458, 0.15165923535823822, 0.003539716824889183, 0.0046739280223846436, 0.16402718424797058, 0.05968156456947327, 0.004154939670115709, 0.010169521905481815, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0662880688905716, 0.04092540964484215, 0.02732268162071705, 0.04505367949604988, 0.06575217843055725, 0.015657562762498856, 0.02788187935948372, 0.013711260631680489, 0.03174294903874397, 0.05280746519565582, 0.015447776764631271, 0.032181113958358765, 0.032901059836149216, 0.01629886031150818, 0.0266366358846426, 0.09772849082946777, 0.017114203423261642, 0.01691945269703865, 0.09847065806388855, 0.04787908494472504, 0.018412545323371887, 0.015331599861383438, 0.17753532528877258, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09801430255174637, 0.026328807696700096, 0.03586815297603607, 0.05713058263063431, 0.06943874061107635, 0.012128958478569984, 0.020909087732434273, 0.008747960440814495, 0.02699965611100197, 0.037266768515110016, 0.011716341599822044, 0.08499539643526077, 0.05501333996653557, 0.012196713127195835, 0.03563909977674484, 0.08789661526679993, 0.012970106676220894, 0.010851015336811543, 0.06408385187387466, 0.059639204293489456, 0.01449217926710844, 0.006093734409660101, 0.09761437773704529, 0.053964968770742416, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02769325114786625, 0.04073771461844444, 0.031223542988300323, 0.058790355920791626, 0.056018248200416565, 0.0015714796027168632, 0.011908041313290596, 0.003339792136102915, 0.051765501499176025, 0.05291244387626648, 0.0018259818898513913, 0.060300350189208984, 0.028357185423374176, 0.0020878082141280174, 0.05086478963494301, 0.07898838073015213, 0.0023987055756151676, 0.002904822351410985, 0.11135734617710114, 0.04586018621921539, 0.0028849171940237284, 0.011431250721216202, 0.1234055683016777, 0.13488474488258362, 0.006487556733191013, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15390776097774506, 0.023786796256899834, 0.05320245772600174, 0.03963885083794594, 0.05711062252521515, 0.015628937631845474, 0.02020607329905033, 0.009252368472516537, 0.02131185494363308, 0.02902994491159916, 0.014741902239620686, 0.0612383633852005, 0.10112591087818146, 0.015157817862927914, 0.04335402697324753, 0.04090719670057297, 0.01573265716433525, 0.020073410123586655, 0.053749360144138336, 0.03468616306781769, 0.01698937825858593, 0.006368495989590883, 0.04042017087340355, 0.031060419976711273, 0.013937168754637241, 0.06738192588090897, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08555091917514801, 0.035057418048381805, 0.05885257571935654, 0.03497908264398575, 0.05178743600845337, 0.0025119921192526817, 0.0041512190364301205, 0.0020859173964709044, 0.014081120491027832, 0.02449135296046734, 0.0023760052863508463, 0.09535116702318192, 0.07766135036945343, 0.0024908811319619417, 0.045726776123046875, 0.05774116516113281, 0.002660380443558097, 0.0033477561082690954, 0.047998279333114624, 0.02678881771862507, 0.0029587154276669025, 0.004655787255614996, 0.03607099875807762, 0.021599192172288895, 0.004788633435964584, 0.22052206099033356, 0.03371305391192436, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.020706716924905777, 0.037194062024354935, 0.01972370594739914, 0.038942236453294754, 0.03182699903845787, 0.0012253678869456053, 0.008815747685730457, 0.002405293518677354, 0.0345795564353466, 0.047298070043325424, 0.001395374652929604, 0.05148206278681755, 0.02048674412071705, 0.0015608376124873757, 0.03826117143034935, 0.04863753169775009, 0.0017825308023020625, 0.0025446040090173483, 0.06541350483894348, 0.0418856181204319, 0.0020960604306310415, 0.009401390329003334, 0.08460027724504471, 0.09019991755485535, 0.0054032751359045506, 0.22932568192481995, 0.05692270025610924, 0.005882916506379843, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08582809567451477, 0.039671752601861954, 0.014841826632618904, 0.056536220014095306, 0.05960613489151001, 0.0069875167682766914, 0.010110098868608475, 0.002887725830078125, 0.015322853811085224, 0.02592875435948372, 0.006830447353422642, 0.033548954874277115, 0.021093523129820824, 0.007213158532977104, 0.019323380663990974, 0.05298228561878204, 0.007658614311367273, 0.005157842300832272, 0.06523183733224869, 0.018170006573200226, 0.008534705266356468, 0.0043681650422513485, 0.026912428438663483, 0.049627941101789474, 0.004404331557452679, 0.19388671219348907, 0.03933224827051163, 0.005363550037145615, 0.11263895034790039, 0.0, 0.0, 0.0, 0.0], [0.07040347903966904, 0.05926162749528885, 0.015334847383201122, 0.0241390373557806, 0.015728840604424477, 0.019492102786898613, 0.017311513423919678, 0.017303066328167915, 0.020563244819641113, 0.0451558418571949, 0.021815825253725052, 0.07079727947711945, 0.030063964426517487, 0.02354060858488083, 0.020684480667114258, 0.02116367779672146, 0.02644512802362442, 0.023540044203400612, 0.05663726478815079, 0.0336947925388813, 0.030707288533449173, 0.015993360430002213, 0.015474079176783562, 0.030803624540567398, 0.03180558606982231, 0.07969609647989273, 0.03437487408518791, 0.034537144005298615, 0.030624371021986008, 0.06290692836046219, 0.0, 0.0, 0.0], [0.05338500067591667, 0.029621528461575508, 0.021030094474554062, 0.023695707321166992, 0.033436402678489685, 0.01839382015168667, 0.024060707539319992, 0.02281002327799797, 0.013972364366054535, 0.028813544660806656, 0.019059743732213974, 0.015371965244412422, 0.02533530630171299, 0.019971953704953194, 0.014075737446546555, 0.0388445146381855, 0.021346209570765495, 0.019321434199810028, 0.056326594203710556, 0.021360008046030998, 0.023394575342535973, 0.023136639967560768, 0.06416381895542145, 0.06724688410758972, 0.0346950888633728, 0.043490175157785416, 0.03032725676894188, 0.03285712003707886, 0.053275078535079956, 0.0659981295466423, 0.041182488203048706, 0.0, 0.0], [0.18177089095115662, 0.019533155485987663, 0.028002941980957985, 0.02911018393933773, 0.04134833440184593, 0.01801948994398117, 0.010471724905073643, 0.008382909931242466, 0.01533578522503376, 0.021839451044797897, 0.01689334586262703, 0.03806138411164284, 0.03643865883350372, 0.017464255914092064, 0.026129068806767464, 0.04011504724621773, 0.018317215144634247, 0.012488218024373055, 0.05136970803141594, 0.01944384165108204, 0.019889770075678825, 0.0035764521453529596, 0.026232963427901268, 0.018429189920425415, 0.011723755858838558, 0.0327298603951931, 0.034808199852705, 0.01203504391014576, 0.06030292063951492, 0.00787330698221922, 0.042272936552762985, 0.07959000021219254, 0.0], [0.015003404580056667, 0.02033485472202301, 0.007813435047864914, 0.016553575173020363, 0.01646609604358673, 0.00023809655976947397, 0.0019598929211497307, 0.0006898260326124728, 0.006483218166977167, 0.010734074749052525, 0.0002443529956508428, 0.014634673483669758, 0.00924762524664402, 0.0002605919726192951, 0.009339917451143265, 0.042726580053567886, 0.0002886394504457712, 0.0003811670758295804, 0.054749373346567154, 0.09452768415212631, 0.00034211305319331586, 0.001727019902318716, 0.04745933413505554, 0.02661379985511303, 0.0010238662362098694, 0.1653798222541809, 0.014187106862664223, 0.0013092352310195565, 0.04770654812455177, 0.0111984983086586, 0.2205076366662979, 0.1394261121749878, 0.0004418626776896417]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9333657622337341, 0.06663428246974945, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.32452359795570374, 0.5923718214035034, 0.08310455828905106, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1415974646806717, 0.514380931854248, 0.28144142031669617, 0.06258013844490051, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.25262272357940674, 0.058340705931186676, 0.10458870232105255, 0.4411159157752991, 0.14333190023899078, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.18051563203334808, 0.09828907996416092, 0.10873956233263016, 0.12262694537639618, 0.09995546191930771, 0.38987332582473755, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1719229817390442, 0.023320995271205902, 0.12332706153392792, 0.06076495721936226, 0.05971711874008179, 0.347591757774353, 0.21335510909557343, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09627032279968262, 0.04683619737625122, 0.028257304802536964, 0.03239249438047409, 0.0586504191160202, 0.15829356014728546, 0.3099648952484131, 0.2693347930908203, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09948235750198364, 0.0646418035030365, 0.028671829029917717, 0.0730566680431366, 0.02978716976940632, 0.1396002322435379, 0.20754705369472504, 0.265802264213562, 0.09141054004430771, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05632037669420242, 0.029885854572057724, 0.01341991126537323, 0.0024075533729046583, 0.013245469890534878, 0.045645419508218765, 0.05522569268941879, 0.10027101635932922, 0.6579986810684204, 0.02558007836341858, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04266994819045067, 0.010426479391753674, 0.010652213357388973, 0.013446621596813202, 0.00935314130038023, 0.04015978425741196, 0.0710517093539238, 0.1373765617609024, 0.10423263162374496, 0.21736638247966766, 0.34326446056365967, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0296839140355587, 0.002561642089858651, 0.004407483618706465, 0.034248705953359604, 0.014238743111491203, 0.028001822531223297, 0.02228802628815174, 0.07105964422225952, 0.04026233032345772, 0.5543041825294495, 0.17333054542541504, 0.025612974539399147, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04039592295885086, 0.008631582371890545, 0.005494222976267338, 0.003864138852804899, 0.023363104090094566, 0.027207026258111, 0.025677336379885674, 0.050485219806432724, 0.0510561428964138, 0.05041782185435295, 0.14164337515830994, 0.511528491973877, 0.060235634446144104, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.028706390410661697, 0.004808460362255573, 0.004723656922578812, 0.005382601171731949, 0.003456039587035775, 0.014470163732767105, 0.024037377908825874, 0.041626207530498505, 0.03567296266555786, 0.07060738652944565, 0.10830992460250854, 0.05811614543199539, 0.2062121331691742, 0.3938705325126648, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05342644825577736, 0.011923279613256454, 0.015572259202599525, 0.0021123222541064024, 0.005851038731634617, 0.02217092178761959, 0.03818469122052193, 0.02741156704723835, 0.054642871022224426, 0.04910830780863762, 0.12328378856182098, 0.05287173017859459, 0.07545649260282516, 0.3701530694961548, 0.09783118963241577, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.008346711285412312, 0.00027852741186507046, 0.0011785993119701743, 0.0021395485382527113, 0.002476169029250741, 0.002918292535468936, 0.004576893523335457, 0.0039655971340835094, 0.004973022732883692, 0.011347771622240543, 0.013014075346291065, 0.005545974709093571, 0.02089560590684414, 0.03838539123535156, 0.8750316500663757, 0.004926194902509451, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02170032449066639, 0.002678664866834879, 0.0023839243222028017, 0.002815627260133624, 0.001556233735755086, 0.006139412056654692, 0.009039806202054024, 0.01374087855219841, 0.012412214651703835, 0.02468654327094555, 0.03406905382871628, 0.017561480402946472, 0.06082957610487938, 0.12004052847623825, 0.11516803503036499, 0.12663689255714417, 0.42854079604148865, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01968139037489891, 0.0028661887627094984, 0.002752861240878701, 0.0023642012383788824, 0.0022205759305506945, 0.0047769746743142605, 0.005015391390770674, 0.007020775228738785, 0.006603003013879061, 0.01582488976418972, 0.02177913300693035, 0.02237590402364731, 0.03905554488301277, 0.07342638820409775, 0.07227564603090286, 0.0810362920165062, 0.2556319534778595, 0.36529290676116943, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.034438684582710266, 0.0024295076727867126, 0.0027038780972361565, 0.0010108931455761194, 0.0028212429024279118, 0.009310661815106869, 0.00747407041490078, 0.010526392608880997, 0.01934259757399559, 0.006583388429135084, 0.030525274574756622, 0.008166542276740074, 0.023945646360516548, 0.08047989010810852, 0.04428938403725624, 0.048865143209695816, 0.23266319930553436, 0.38709720969200134, 0.04732644557952881, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0036976831033825874, 0.006450952962040901, 0.0004921727231703699, 0.0003069589729420841, 0.00012400773994158953, 0.0010727370390668511, 0.0008858796209096909, 0.001380636473186314, 0.0008200175943784416, 0.0002731183485593647, 0.002783065428957343, 0.03529426082968712, 0.0035613791551440954, 0.007148314733058214, 0.001647408353164792, 0.016731660813093185, 0.01917281188070774, 0.022339196875691414, 0.8733668327331543, 0.002450960921123624, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.013628787361085415, 0.001438445644453168, 0.0012188641121611, 0.0012915851548314095, 0.0006118023884482682, 0.0021400381810963154, 0.0027469571214169264, 0.0033360477536916733, 0.003091872902587056, 0.005988912656903267, 0.007237219717353582, 0.003547744592651725, 0.012891951948404312, 0.022229159250855446, 0.020824221894145012, 0.02629532851278782, 0.0764816403388977, 0.16261443495750427, 0.1263866275548935, 0.10034442692995071, 0.4056538939476013, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.021432969719171524, 0.0007291649235412478, 0.0010889691766351461, 0.0008559620473533869, 0.0010556603083387017, 0.002354251453652978, 0.0018633343279361725, 0.002486992860212922, 0.004011375363916159, 0.0036456233356148005, 0.006636620499193668, 0.0024591651745140553, 0.00810664240270853, 0.01950138807296753, 0.013945515267550945, 0.01683160476386547, 0.06727856397628784, 0.1020374447107315, 0.07506179064512253, 0.03785506263375282, 0.3474159240722656, 0.2633460462093353, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01837441883981228, 0.0016340153524652123, 0.0002648660447448492, 0.0006277099018916488, 0.0007212890777736902, 0.0022947092074900866, 0.0014375777682289481, 0.0024335708003491163, 0.0010225051082670689, 0.002173094777390361, 0.005665970034897327, 0.000585706380661577, 0.004730311222374439, 0.01578962802886963, 0.009121065959334373, 0.006447889842092991, 0.049651872366666794, 0.07221098989248276, 0.04530426487326622, 0.02253621071577072, 0.23380489647388458, 0.28100958466529846, 0.22215785086154938, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01601027511060238, 0.00041907597915269434, 0.0016793111572042108, 0.0020331626292318106, 0.0011126026511192322, 0.0023041211534291506, 0.001641488284803927, 0.0020452297758311033, 0.0008826447883620858, 0.003531813621520996, 0.00466074887663126, 0.001146126538515091, 0.004025280009955168, 0.011729470454156399, 0.006906012073159218, 0.005978981498628855, 0.03413612022995949, 0.053919851779937744, 0.07536919414997101, 0.015544144436717033, 0.153072789311409, 0.21672524511814117, 0.23267339169979095, 0.1524529755115509, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01641550101339817, 0.0010881004855036736, 0.001302703283727169, 0.001233565155416727, 0.0010325239272788167, 0.0013605316635221243, 0.001135275000706315, 0.0009713207837194204, 0.0017580230487510562, 0.002354995347559452, 0.0022677984088659286, 0.0026170071214437485, 0.0032849612180143595, 0.00556444376707077, 0.0072877430357038975, 0.006582668051123619, 0.0169111005961895, 0.023348137736320496, 0.033363793045282364, 0.01206086203455925, 0.08410514146089554, 0.10776982456445694, 0.12741318345069885, 0.2500958740711212, 0.2886749505996704, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.031117741018533707, 0.0015063054161146283, 0.0010839324677363038, 0.002421524142846465, 0.001382686896249652, 0.0037937266752123833, 0.001633929554373026, 0.003119518980383873, 0.0012812635395675898, 0.002068523084744811, 0.0047601195983588696, 0.0008757320465520024, 0.008592123165726662, 0.0093536376953125, 0.011993243359029293, 0.005963001865893602, 0.022028174251317978, 0.03411327302455902, 0.009586759842932224, 0.013681814074516296, 0.08417054265737534, 0.08917812258005142, 0.10806837677955627, 0.15560784935951233, 0.37236863374710083, 0.020249491557478905, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.014838481321930885, 0.0005757318576797843, 0.004743653815239668, 0.00045677347225137055, 0.0007412288105115294, 0.0015000010607764125, 0.0008866073912940919, 0.0008452099282294512, 0.00010359263978898525, 0.000906514236703515, 0.0018096797866746783, 0.0602911040186882, 0.001537756179459393, 0.0033079395070672035, 0.0013713664375245571, 0.002082675462588668, 0.008643986657261848, 0.009088865481317043, 0.007760955020785332, 0.00580833712592721, 0.033865127712488174, 0.03987184166908264, 0.05374552309513092, 0.036947719752788544, 0.12259436398744583, 0.5718704462051392, 0.013804465532302856, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01857001520693302, 0.0012143177445977926, 0.001288044499233365, 0.0010612568585202098, 0.0007490046555176377, 0.001048562815412879, 0.000868787697982043, 0.0005973864463157952, 0.0012938270810991526, 0.001404251204803586, 0.0011055096983909607, 0.0005003446131013334, 0.0020262624602764845, 0.0021463141310960054, 0.0028737925458699465, 0.0026413712184876204, 0.005802977364510298, 0.0081913648173213, 0.00833908375352621, 0.005543305538594723, 0.026558682322502136, 0.03308920934796333, 0.032507359981536865, 0.09199848771095276, 0.10410483181476593, 0.16861778497695923, 0.208954319357872, 0.2669035494327545, 0.0, 0.0, 0.0, 0.0, 0.0], [0.016654442995786667, 0.0007248665788210928, 0.0008980666170828044, 0.0031616878695786, 0.0005522824940271676, 0.0016607288271188736, 0.00074861227767542, 0.0009652797016315162, 0.0003842804580926895, 0.0022577813360840082, 0.0015545616624876857, 0.0007037006434984505, 0.0014688181690871716, 0.002799882087856531, 0.003013978712260723, 0.005369038321077824, 0.006552346982061863, 0.008701568469405174, 0.007138868793845177, 0.006516328547149897, 0.026185575872659683, 0.03929203003644943, 0.03155006840825081, 0.02434305101633072, 0.1067076250910759, 0.15159794688224792, 0.13674212992191315, 0.3201395273208618, 0.09161490201950073, 0.0, 0.0, 0.0, 0.0], [0.0372268483042717, 0.0007442747591994703, 0.0019180340459570289, 0.0014290640829131007, 0.0025597368367016315, 0.003292836481705308, 0.0009007275220938027, 0.001544511178508401, 0.0003557214804459363, 0.0004492733278311789, 0.0026688158977776766, 0.0014046661090105772, 0.0017281017499044538, 0.004542736802250147, 0.0012941033346578479, 0.004677668213844299, 0.009138188324868679, 0.008992945775389671, 0.005343662574887276, 0.0020318389870226383, 0.030930904671549797, 0.021215269342064857, 0.03639978542923927, 0.056794628500938416, 0.11195950210094452, 0.011458109132945538, 0.03353285789489746, 0.2658853828907013, 0.2469542920589447, 0.09262555837631226, 0.0, 0.0, 0.0], [0.044088784605264664, 0.0011082949349656701, 0.0033875906374305487, 0.004853191319853067, 0.002864320995286107, 0.004500862676650286, 0.0014518116367980838, 0.0021924942266196012, 0.0007902359939180315, 0.0006717380601912737, 0.003577157622203231, 0.0005131773068569601, 0.012861719354987144, 0.0051168459467589855, 0.0030142527539283037, 0.005887466017156839, 0.010453412309288979, 0.013818331062793732, 0.008691059425473213, 0.011229269206523895, 0.0314009003341198, 0.023181650787591934, 0.03044436313211918, 0.03520682081580162, 0.09999866038560867, 0.006090165581554174, 0.14714214205741882, 0.16393053531646729, 0.12749530375003815, 0.1935775727033615, 0.0004599907260853797, 0.0, 0.0], [0.03717399016022682, 0.005006894003599882, 0.0016112301964312792, 0.0028301470447331667, 0.0005118494154885411, 0.0027071270160377026, 0.0020970311015844345, 0.0012328773736953735, 0.0006310884491540492, 0.001763244392350316, 0.001786878565326333, 0.00025283300783485174, 0.0010720754507929087, 0.0026448338758200407, 0.0015661576762795448, 0.0038084082771092653, 0.005276741925626993, 0.006930254865437746, 0.01331083569675684, 0.0035816438030451536, 0.018304521217942238, 0.026684748008847237, 0.019971538335084915, 0.03215576708316803, 0.07432040572166443, 0.023922255262732506, 0.08761478215456009, 0.1800757199525833, 0.027492573484778404, 0.059722453355789185, 0.19122688472270966, 0.1627122461795807, 0.0], [0.00811693910509348, 0.0008723069913685322, 0.0003707293944898993, 0.00047348643420264125, 0.0001783298939699307, 0.0003994710568804294, 0.00029511720640584826, 0.00018311945314053446, 0.000167145932209678, 0.00023702479666098952, 0.0002533360675442964, 0.00012855060049332678, 0.00020213205425534397, 0.0003994489670731127, 0.0001651181373745203, 0.00046228704741224647, 0.0008780225180089474, 0.0011140474816784263, 0.0020708786323666573, 0.0019636773504316807, 0.0035278573632240295, 0.0044752187095582485, 0.004463486839085817, 0.007312532514333725, 0.014446244575083256, 0.035030465573072433, 0.02001415751874447, 0.04388578608632088, 0.019436758011579514, 0.11325737833976746, 0.07277627289295197, 0.16196827590465546, 0.4804745018482208]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.947374701499939, 0.05262523517012596, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8785883784294128, 0.055394671857357025, 0.06601690500974655, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7270770072937012, 0.1370987594127655, 0.04866093024611473, 0.08716330677270889, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4651746451854706, 0.13429315388202667, 0.1535816341638565, 0.1747254729270935, 0.07222513109445572, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1703597903251648, 0.011841950938105583, 0.0236508846282959, 0.021745815873146057, 0.025856325402855873, 0.7465452551841736, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14214572310447693, 0.04383273795247078, 0.050211820751428604, 0.04516506567597389, 0.058721259236335754, 0.5111533999443054, 0.1487700343132019, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0847334936261177, 0.020369820296764374, 0.03746161609888077, 0.03195912018418312, 0.03419598937034607, 0.43500685691833496, 0.1189405769109726, 0.2373325079679489, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3423079550266266, 0.03593634441494942, 0.0725921243429184, 0.10352761298418045, 0.08196527510881424, 0.07694562524557114, 0.14196324348449707, 0.09896710515022278, 0.045794595032930374, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3711485266685486, 0.06278738379478455, 0.05806950107216835, 0.055662769824266434, 0.13124839961528778, 0.07535629719495773, 0.10726629197597504, 0.07924024015665054, 0.030107343569397926, 0.029113225638866425, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.06430108100175858, 0.0051691061817109585, 0.01101299561560154, 0.009054888971149921, 0.011748063378036022, 0.3188501000404358, 0.03684947267174721, 0.0831955149769783, 0.012068143114447594, 0.014676210470497608, 0.433074414730072, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.23777922987937927, 0.03812997415661812, 0.01974336989223957, 0.07508028298616409, 0.14420795440673828, 0.06244930252432823, 0.07018385827541351, 0.06523600965738297, 0.048832181841135025, 0.03460032865405083, 0.06439767777919769, 0.13935980200767517, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.29014477133750916, 0.04085766524076462, 0.06965630501508713, 0.06467737257480621, 0.11515594273805618, 0.07279439270496368, 0.05406038090586662, 0.040959976613521576, 0.036461226642131805, 0.03709695488214493, 0.08112155646085739, 0.06249447911977768, 0.03451891988515854, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04191606491804123, 0.0033880723640322685, 0.007082367315888405, 0.005899207200855017, 0.007586182560771704, 0.20389790832996368, 0.023296764120459557, 0.05173072963953018, 0.007816805504262447, 0.00973137654364109, 0.2776019871234894, 0.008386771194636822, 0.01617271639406681, 0.3354930877685547, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15129348635673523, 0.04240793362259865, 0.043833814561367035, 0.09706760942935944, 0.14943556487560272, 0.044398821890354156, 0.0617450475692749, 0.04199514910578728, 0.018608780577778816, 0.031412459909915924, 0.05092642828822136, 0.06660290062427521, 0.09416216611862183, 0.057612884789705276, 0.04849693179130554, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.29958024621009827, 0.07432066649198532, 0.03006197139620781, 0.016736475750803947, 0.038564808666706085, 0.05106877535581589, 0.09408668428659439, 0.05606791749596596, 0.054582275450229645, 0.045220401138067245, 0.056380707770586014, 0.015378053300082684, 0.04082852229475975, 0.06084694340825081, 0.030811049044132233, 0.03546451777219772, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.030439963564276695, 0.0024933936074376106, 0.005298204720020294, 0.004223393276333809, 0.005517445504665375, 0.14508457481861115, 0.01603415422141552, 0.03598647937178612, 0.0057580480352044106, 0.007398170419037342, 0.19613467156887054, 0.006090890150517225, 0.011465118266642094, 0.23737992346286774, 0.008031771518290043, 0.005861491896212101, 0.2768023610115051, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015668945387005806, 0.0019133484456688166, 0.006747933570295572, 0.005610871594399214, 0.006710356567054987, 0.10858529061079025, 0.013436974957585335, 0.030674796551465988, 0.00869804061949253, 0.005958769004791975, 0.15696409344673157, 0.00640126271173358, 0.01435719896107912, 0.19220466911792755, 0.009938806295394897, 0.008154557086527348, 0.22861403226852417, 0.1793600618839264, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2751207649707794, 0.046356070786714554, 0.025782747194170952, 0.02037108689546585, 0.04519129917025566, 0.047415051609277725, 0.05800836905837059, 0.054494574666023254, 0.02185583859682083, 0.028674187138676643, 0.05054797977209091, 0.03578946739435196, 0.014343999326229095, 0.05505189672112465, 0.019970891997218132, 0.0250706784427166, 0.05983670800924301, 0.10591268539428711, 0.01020571868866682, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12210460752248764, 0.029006367549300194, 0.02792610600590706, 0.030763691291213036, 0.06431955099105835, 0.05075869709253311, 0.05746712535619736, 0.03854454681277275, 0.02881445176899433, 0.03641033545136452, 0.05530688911676407, 0.02755570411682129, 0.05062380060553551, 0.06039686128497124, 0.03790910169482231, 0.034162603318691254, 0.06550325453281403, 0.08224605768918991, 0.08903136104345322, 0.011148839257657528, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01992272026836872, 0.0015991260297596455, 0.003536272095516324, 0.0027606096118688583, 0.0036747483536601067, 0.09198574721813202, 0.009868193417787552, 0.021762730553746223, 0.0036746191326528788, 0.004800298251211643, 0.12327182292938232, 0.003948224242776632, 0.007727704476565123, 0.14901942014694214, 0.005094495136290789, 0.0038495466578751802, 0.17433811724185944, 0.1450374871492386, 0.003139176871627569, 0.0039015670772641897, 0.21708732843399048, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.025169141590595245, 0.0025891433469951153, 0.007904480211436749, 0.002818175358697772, 0.008317488245666027, 0.06284766644239426, 0.016097376123070717, 0.024833567440509796, 0.007658041547983885, 0.011317074298858643, 0.08451006561517715, 0.012135406024754047, 0.022048376500606537, 0.1001725122332573, 0.015103276818990707, 0.020152254030108452, 0.11906814575195312, 0.141489639878273, 0.006576653569936752, 0.007074056193232536, 0.1485065221786499, 0.1536109447479248, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08679644763469696, 0.01581868715584278, 0.009913153015077114, 0.005941125564277172, 0.01174909994006157, 0.04402247071266174, 0.0494115874171257, 0.047211576253175735, 0.022448517382144928, 0.02814415469765663, 0.05249372497200966, 0.013664884492754936, 0.0380445197224617, 0.05856490507721901, 0.01600872538983822, 0.028531765565276146, 0.06752399355173111, 0.1000405102968216, 0.026074493303894997, 0.01816527359187603, 0.08061447739601135, 0.14588385820388794, 0.03293200954794884, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12203323096036911, 0.02679692953824997, 0.03471272066235542, 0.016600849106907845, 0.05098596587777138, 0.033623479306697845, 0.03376290947198868, 0.03289441019296646, 0.022763589397072792, 0.03469543159008026, 0.03710823133587837, 0.02750304900109768, 0.030508041381835938, 0.04070492461323738, 0.019409826025366783, 0.033388134092092514, 0.046037882566452026, 0.08141940832138062, 0.025792937725782394, 0.017713353037834167, 0.05326378345489502, 0.06955216079950333, 0.07340100407600403, 0.03532780334353447, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.012632706202566624, 0.003196730511263013, 0.00899962242692709, 0.003787106368690729, 0.0055902148596942425, 0.060187432914972305, 0.010173025541007519, 0.026063401252031326, 0.008280332200229168, 0.005998437758535147, 0.08480245620012283, 0.007755732629448175, 0.014807655476033688, 0.10079342871904373, 0.011636020615696907, 0.007885102182626724, 0.120613694190979, 0.12819863855838776, 0.006487111561000347, 0.00323057291097939, 0.15047506988048553, 0.10557688027620316, 0.015480604022741318, 0.01902633160352707, 0.0783216655254364, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11176053434610367, 0.018635563552379608, 0.05622374638915062, 0.046772800385951996, 0.03736491501331329, 0.02299342304468155, 0.031365279108285904, 0.026408886536955833, 0.043359220027923584, 0.056635573506355286, 0.021786397323012352, 0.023822149261832237, 0.030641673132777214, 0.023289257660508156, 0.05430489778518677, 0.06881187111139297, 0.024489382281899452, 0.04777085781097412, 0.036271754652261734, 0.022222526371479034, 0.026998935267329216, 0.02176610752940178, 0.03442488983273506, 0.028240257874131203, 0.05891920626163483, 0.024719828739762306, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10639805346727371, 0.0371079258620739, 0.08273956179618835, 0.01785685308277607, 0.020949997007846832, 0.018305253237485886, 0.02262011356651783, 0.018192270770668983, 0.021851180121302605, 0.01251852884888649, 0.019214097410440445, 0.03034009225666523, 0.0718761533498764, 0.020787999033927917, 0.03963525965809822, 0.06151590868830681, 0.02317928522825241, 0.0439804382622242, 0.04619227349758148, 0.02066798508167267, 0.027080876752734184, 0.04386550188064575, 0.0327674001455307, 0.04369068145751953, 0.05080847069621086, 0.04029490798711777, 0.025563018396496773, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015084554441273212, 0.003822384402155876, 0.005877268500626087, 0.0049718520604074, 0.005370147991925478, 0.05223645642399788, 0.016127675771713257, 0.03182988613843918, 0.00821112934499979, 0.011473809368908405, 0.06704685091972351, 0.010528381913900375, 0.010598219931125641, 0.07938898354768753, 0.007211714517325163, 0.0062714628875255585, 0.09142373502254486, 0.10866505652666092, 0.003973816987127066, 0.005621673539280891, 0.1098332554101944, 0.09237755835056305, 0.02296048402786255, 0.019314926117658615, 0.08864143490791321, 0.018464695662260056, 0.011022249236702919, 0.0916503295302391, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09854131937026978, 0.03967057913541794, 0.05052143707871437, 0.04794390872120857, 0.0195318553596735, 0.015683406963944435, 0.02507476694881916, 0.014997915364801884, 0.02772345580160618, 0.016418185085058212, 0.016389017924666405, 0.026316562667489052, 0.03435927629470825, 0.017757989466190338, 0.0233626626431942, 0.03763514757156372, 0.019405605271458626, 0.034421682357788086, 0.04902297630906105, 0.030190544202923775, 0.022159066051244736, 0.037540629506111145, 0.038793135434389114, 0.04032415151596069, 0.0423126220703125, 0.06829195469617844, 0.025314129889011383, 0.04330073297023773, 0.03699527308344841, 0.0, 0.0, 0.0, 0.0], [0.10688814520835876, 0.03659575805068016, 0.005083094350993633, 0.01016042660921812, 0.013297352008521557, 0.04175207018852234, 0.0262468159198761, 0.03012855537235737, 0.013482386246323586, 0.018223188817501068, 0.0458262674510479, 0.02555879019200802, 0.015618832781910896, 0.05039895698428154, 0.01165701076388359, 0.011659128591418266, 0.0551551915705204, 0.0509166419506073, 0.019206784665584564, 0.013998407870531082, 0.06383473426103592, 0.02908892184495926, 0.016710441559553146, 0.02609526738524437, 0.05484986677765846, 0.03806670755147934, 0.03896071016788483, 0.05375232174992561, 0.023679165169596672, 0.0531080923974514, 0.0, 0.0, 0.0], [0.10871604084968567, 0.06286415457725525, 0.02304510399699211, 0.012493208050727844, 0.009688262827694416, 0.02372078038752079, 0.025617338716983795, 0.028282862156629562, 0.019523099064826965, 0.015187123790383339, 0.024615760892629623, 0.018855812028050423, 0.013071928173303604, 0.02569747343659401, 0.010639789514243603, 0.007336440030485392, 0.028488244861364365, 0.040875114500522614, 0.023693937808275223, 0.02846083790063858, 0.031781155616045, 0.03399210050702095, 0.01690451242029667, 0.04699751362204552, 0.054809220135211945, 0.0604817196726799, 0.021137842908501625, 0.05147771164774895, 0.01530434750020504, 0.07868867367506027, 0.03755194693803787, 0.0, 0.0], [0.16510623693466187, 0.013307320885360241, 0.022820336744189262, 0.032224975526332855, 0.028327813372015953, 0.011912115849554539, 0.021759122610092163, 0.012591322883963585, 0.025371331721544266, 0.01592349074780941, 0.011745063588023186, 0.021340155974030495, 0.022658517584204674, 0.012096998281776905, 0.05877390503883362, 0.041142962872982025, 0.012866133823990822, 0.02189669944345951, 0.02738575078547001, 0.015412718988955021, 0.014822259545326233, 0.013198777101933956, 0.029060106724500656, 0.04031305015087128, 0.023036926984786987, 0.05035790055990219, 0.046500690281391144, 0.03221071884036064, 0.04574136063456535, 0.011448168195784092, 0.05148483067750931, 0.04716223478317261, 0.0], [0.019862741231918335, 0.0006941381143406034, 0.0017670642118901014, 0.0019038202008232474, 0.0023115782532840967, 0.05924388766288757, 0.005164953414350748, 0.008714546449482441, 0.0024906685575842857, 0.0030579573940485716, 0.07585158944129944, 0.002390549285337329, 0.0031773503869771957, 0.08945121616125107, 0.0030652948189526796, 0.0016209169989451766, 0.10312031954526901, 0.0625915676355362, 0.002058295765891671, 0.002232219558209181, 0.12742049992084503, 0.024703850969672203, 0.0035101925022900105, 0.00499417819082737, 0.028317121788859367, 0.005874201189726591, 0.006215325556695461, 0.03686564415693283, 0.005438121035695076, 0.004061868414282799, 0.004735523369163275, 0.004534382838755846, 0.2925584614276886]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9726406335830688, 0.027359364554286003, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8850566744804382, 0.08160645514726639, 0.03333686292171478, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7844979166984558, 0.09045999497175217, 0.10720972716808319, 0.01783234067261219, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6679065227508545, 0.10135099291801453, 0.10997921228408813, 0.0699087604880333, 0.050854478031396866, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.586279034614563, 0.06960930675268173, 0.06798022985458374, 0.06058962643146515, 0.08339767158031464, 0.13214409351348877, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5015710592269897, 0.06227877363562584, 0.07209988683462143, 0.05590919405221939, 0.08146880567073822, 0.12033165246248245, 0.10634062439203262, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.40016087889671326, 0.05525398626923561, 0.06347683072090149, 0.05413786694407463, 0.07119617611169815, 0.11230527609586716, 0.12489721924066544, 0.11857166141271591, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4018532931804657, 0.08579416573047638, 0.056448839604854584, 0.07181122153997421, 0.08318912237882614, 0.07845646142959595, 0.09191881120204926, 0.07768876850605011, 0.05283927917480469, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3827159106731415, 0.07976946234703064, 0.062077559530735016, 0.06360235810279846, 0.08211023360490799, 0.07804954797029495, 0.08597437292337418, 0.07506486773490906, 0.0632784441113472, 0.02735721692442894, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3404167592525482, 0.04535197094082832, 0.04658910632133484, 0.0403917022049427, 0.05489042401313782, 0.08179006725549698, 0.08906516432762146, 0.09645897150039673, 0.05564969405531883, 0.06398677080869675, 0.08540938794612885, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.31186187267303467, 0.09539996832609177, 0.07292013615369797, 0.0697687417268753, 0.07777827978134155, 0.06735038757324219, 0.06161406263709068, 0.054810743778944016, 0.052520137280225754, 0.05022520199418068, 0.07299348711967468, 0.012756960466504097, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.36652863025665283, 0.06308672577142715, 0.03947845473885536, 0.04609525203704834, 0.05740750953555107, 0.0634833425283432, 0.05238465964794159, 0.05501618608832359, 0.04584015905857086, 0.06228693574666977, 0.0658990740776062, 0.058063607662916183, 0.024429509416222572, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.28564438223838806, 0.03804025053977966, 0.03942390903830528, 0.034238625317811966, 0.0463939793407917, 0.06771662831306458, 0.07328074425458908, 0.07855433225631714, 0.047896627336740494, 0.055936526507139206, 0.07124947756528854, 0.035689257085323334, 0.04956042021512985, 0.07637479156255722, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.297754168510437, 0.05259547755122185, 0.04347941279411316, 0.05845862627029419, 0.04568730294704437, 0.05259405076503754, 0.04456199333071709, 0.0459747239947319, 0.06316147744655609, 0.0776875764131546, 0.05499143525958061, 0.03466471657156944, 0.05020103231072426, 0.05886140838265419, 0.01932653971016407, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.28853145241737366, 0.04616911709308624, 0.028233066201210022, 0.038242388516664505, 0.04330621287226677, 0.05174916982650757, 0.04318275675177574, 0.043931201100349426, 0.05616341903805733, 0.06773821264505386, 0.05490177869796753, 0.08139309287071228, 0.04325690492987633, 0.0590481236577034, 0.03989904746413231, 0.014254068955779076, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.23721373081207275, 0.03161581605672836, 0.03324565663933754, 0.029231945052742958, 0.038618117570877075, 0.05549168214201927, 0.05966496840119362, 0.06321781128644943, 0.040736980736255646, 0.04752674326300621, 0.058378204703330994, 0.03039967454969883, 0.04287158325314522, 0.06284616142511368, 0.053212910890579224, 0.04736898094415665, 0.06835903972387314, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2098696380853653, 0.02729126438498497, 0.031156614422798157, 0.026615004986524582, 0.03165113925933838, 0.05273226648569107, 0.052745021879673004, 0.05635902285575867, 0.04076546058058739, 0.04278237372636795, 0.056191232055425644, 0.030678272247314453, 0.04398961737751961, 0.06091473996639252, 0.0501369833946228, 0.0415731742978096, 0.06652864068746567, 0.07801953703165054, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2385382503271103, 0.04414580762386322, 0.046435050666332245, 0.0345127172768116, 0.0357217937707901, 0.04369329661130905, 0.04304974898695946, 0.034988246858119965, 0.043718066066503525, 0.0395219661295414, 0.044578537344932556, 0.024599451571702957, 0.0625118762254715, 0.04753483459353447, 0.043756093829870224, 0.04995930567383766, 0.050908081233501434, 0.0526624359190464, 0.019164469093084335, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.17729151248931885, 0.042214617133140564, 0.030818725004792213, 0.030886096879839897, 0.033868495374917984, 0.045211877673864365, 0.042901039123535156, 0.032215580344200134, 0.040188658982515335, 0.049268968403339386, 0.04801624268293381, 0.03814242035150528, 0.051265135407447815, 0.05206841975450516, 0.047441162168979645, 0.03924533724784851, 0.05630478635430336, 0.05462942644953728, 0.0599186010658741, 0.02810293436050415, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19048276543617249, 0.025203485041856766, 0.026787471026182175, 0.02367268316447735, 0.03110402822494507, 0.04345344379544258, 0.04649331048130989, 0.048206478357315063, 0.03323280066251755, 0.03892753645777702, 0.04549149051308632, 0.02444273605942726, 0.035701580345630646, 0.04907767102122307, 0.04303564876317978, 0.039055656641721725, 0.0537174753844738, 0.06658957898616791, 0.04320826008915901, 0.03176470100879669, 0.06035127118229866, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14975358545780182, 0.023474570363759995, 0.025508398190140724, 0.028942441567778587, 0.03517158329486847, 0.038801901042461395, 0.04594280198216438, 0.03962729126214981, 0.037087779492139816, 0.038723014295101166, 0.04190719500184059, 0.021688053384423256, 0.04328255355358124, 0.04592541232705116, 0.03893951326608658, 0.0400761142373085, 0.05060023441910744, 0.060830533504486084, 0.043721601366996765, 0.03023521602153778, 0.05728611722588539, 0.062474027276039124, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.17513407766819, 0.027519281953573227, 0.029514014720916748, 0.02071532793343067, 0.026873527094721794, 0.0345461331307888, 0.038530781865119934, 0.033037811517715454, 0.04181556776165962, 0.04010028764605522, 0.03696895390748978, 0.030468937009572983, 0.039953943341970444, 0.04023702070116997, 0.0448262095451355, 0.03708262741565704, 0.04420144855976105, 0.04931037127971649, 0.04125744849443436, 0.028610678389668465, 0.04997344687581062, 0.05269285663962364, 0.036629173904657364, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13285990059375763, 0.040708061307668686, 0.04111678898334503, 0.034844279289245605, 0.04087604954838753, 0.027567215263843536, 0.028293700888752937, 0.022520417347550392, 0.0387541837990284, 0.040378037840127945, 0.029089409857988358, 0.029609227553009987, 0.0567011833190918, 0.031181858852505684, 0.037661172449588776, 0.056886378675699234, 0.03396708890795708, 0.037813056260347366, 0.04951409250497818, 0.031945571303367615, 0.037599027156829834, 0.04326637089252472, 0.043056681752204895, 0.033790234476327896, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14709368348121643, 0.018218709155917168, 0.022143620997667313, 0.02162139303982258, 0.02707289345562458, 0.03385418280959129, 0.03787611797451973, 0.03801965340971947, 0.027932504191994667, 0.03090183064341545, 0.03606922924518585, 0.01902313157916069, 0.03465801104903221, 0.03916986659169197, 0.033755578100681305, 0.02613663300871849, 0.042978327721357346, 0.05313742905855179, 0.03463505581021309, 0.02014029398560524, 0.04878358915448189, 0.04990803450345993, 0.04624392092227936, 0.04715671017765999, 0.06346960365772247, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1760278195142746, 0.01488215196877718, 0.026751182973384857, 0.0240880586206913, 0.03566748648881912, 0.027094293385744095, 0.02498425543308258, 0.021471885964274406, 0.026359373703598976, 0.023845119401812553, 0.027566982433199883, 0.01686953380703926, 0.0777742937207222, 0.029109211638569832, 0.04799698293209076, 0.02900250256061554, 0.03132336959242821, 0.03615599498152733, 0.03959561511874199, 0.017008014023303986, 0.034839075058698654, 0.03443719446659088, 0.04149218648672104, 0.09124380350112915, 0.040522992610931396, 0.0038906026165932417, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1850174218416214, 0.027787016704678535, 0.025097640231251717, 0.027018999680876732, 0.029199207201600075, 0.030337024480104446, 0.032221775501966476, 0.02301507070660591, 0.0267928596585989, 0.022793851792812347, 0.030674852430820465, 0.021220838651061058, 0.053888075053691864, 0.03259299322962761, 0.030737651512026787, 0.025725245475769043, 0.03486606478691101, 0.037935830652713776, 0.028672009706497192, 0.026148982346057892, 0.03875335678458214, 0.04179506003856659, 0.03863157331943512, 0.05337152257561684, 0.03982071578502655, 0.02571161463856697, 0.010172729380428791, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12446325272321701, 0.01676889881491661, 0.02010510489344597, 0.019267024472355843, 0.024457814171910286, 0.03020336478948593, 0.03127393499016762, 0.03254607319831848, 0.023186003789305687, 0.02350839227437973, 0.0317331999540329, 0.017080510035157204, 0.029082054272294044, 0.03428216651082039, 0.03579127788543701, 0.025820568203926086, 0.037531185895204544, 0.04618249833583832, 0.029500003904104233, 0.017913807183504105, 0.04274731129407883, 0.042434703558683395, 0.039002396166324615, 0.04141838476061821, 0.05949503183364868, 0.03595735505223274, 0.030984872952103615, 0.057262834161520004, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14491021633148193, 0.025843990966677666, 0.031176971271634102, 0.01946275867521763, 0.012575727887451649, 0.025274591520428658, 0.022020690143108368, 0.021267279982566833, 0.03165539726614952, 0.02766416408121586, 0.026251589879393578, 0.022091038525104523, 0.03851794824004173, 0.028266867622733116, 0.0288776233792305, 0.036865364760160446, 0.03069997765123844, 0.03204502537846565, 0.03411627188324928, 0.02035541459918022, 0.03446315601468086, 0.03542865440249443, 0.030728410929441452, 0.04441893845796585, 0.03793596476316452, 0.05734865367412567, 0.03889700025320053, 0.04111205041408539, 0.019728200510144234, 0.0, 0.0, 0.0, 0.0], [0.13168425858020782, 0.025460105389356613, 0.02796240895986557, 0.024109845981001854, 0.01686880923807621, 0.031406790018081665, 0.023247336968779564, 0.022071687504649162, 0.027362508699297905, 0.02637045457959175, 0.032045334577560425, 0.02877580001950264, 0.04614202678203583, 0.03455977141857147, 0.03132109344005585, 0.02943773753941059, 0.03749070689082146, 0.03597421571612358, 0.0306710172444582, 0.026778314262628555, 0.041662365198135376, 0.03274398297071457, 0.030348768457770348, 0.032651983201503754, 0.038056500256061554, 0.034575607627630234, 0.022936122491955757, 0.03690328821539879, 0.02596619911491871, 0.01441489439457655, 0.0, 0.0, 0.0], [0.1656370609998703, 0.02280924655497074, 0.029107755050063133, 0.02494465932250023, 0.02435287833213806, 0.02503182739019394, 0.018486661836504936, 0.01857367716729641, 0.028443630784749985, 0.029199941083788872, 0.0253182053565979, 0.03220498561859131, 0.025753282010555267, 0.026890520006418228, 0.031273938715457916, 0.03305148705840111, 0.028835458680987358, 0.029910407960414886, 0.0275588259100914, 0.024624649435281754, 0.0320536270737648, 0.02649376355111599, 0.02170960046350956, 0.031640153378248215, 0.03707476332783699, 0.0479947030544281, 0.02793215773999691, 0.034756116569042206, 0.03804944083094597, 0.02153567597270012, 0.008750851266086102, 0.0, 0.0], [0.16794215142726898, 0.025983478873968124, 0.03222338482737541, 0.019469385966658592, 0.023735277354717255, 0.023115742951631546, 0.0199736300855875, 0.016550231724977493, 0.029796775430440903, 0.028494667261838913, 0.023108843713998795, 0.02208707109093666, 0.042165566235780716, 0.02439018338918686, 0.02074914425611496, 0.038735877722501755, 0.026269812136888504, 0.02728910744190216, 0.019617389887571335, 0.020242862403392792, 0.02920321375131607, 0.030413366854190826, 0.02576262876391411, 0.040849827229976654, 0.033326659351587296, 0.021795300766825676, 0.026310516521334648, 0.03101349249482155, 0.03689264506101608, 0.01937715709209442, 0.04355606809258461, 0.009558655321598053, 0.0], [0.11396828293800354, 0.01775287464261055, 0.01606196165084839, 0.015585055574774742, 0.01839612051844597, 0.024367082864046097, 0.026053881272673607, 0.023999109864234924, 0.021919487044215202, 0.026251113042235374, 0.024730533361434937, 0.015323017723858356, 0.025614570826292038, 0.0263378843665123, 0.024982867762446404, 0.02464262954890728, 0.028590328991413116, 0.03295150771737099, 0.027568619698286057, 0.01848754845559597, 0.03220851346850395, 0.031914692372083664, 0.036307260394096375, 0.0341540202498436, 0.036785583943128586, 0.03297772631049156, 0.026278911158442497, 0.03809891268610954, 0.029801947996020317, 0.018183261156082153, 0.03632068261504173, 0.035680945962667465, 0.05770310387015343]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8058414459228516, 0.19415856897830963, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6373865008354187, 0.09672051668167114, 0.26589304208755493, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5870265364646912, 0.09922721236944199, 0.07913831621408463, 0.23460792005062103, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4194737374782562, 0.06312593817710876, 0.08123067021369934, 0.08381500095129013, 0.35235461592674255, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5308531522750854, 0.0959072858095169, 0.07779339700937271, 0.06661548465490341, 0.07293295115232468, 0.15589767694473267, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.38503071665763855, 0.07478094846010208, 0.06978150457143784, 0.047606538981199265, 0.051449161022901535, 0.08776707202196121, 0.2835841178894043, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2962956726551056, 0.05328270420432091, 0.06944749504327774, 0.04097547382116318, 0.06660831719636917, 0.1021856963634491, 0.10869503766298294, 0.26250961422920227, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.30474990606307983, 0.06337985396385193, 0.05753423646092415, 0.04057719185948372, 0.0730014219880104, 0.07375827431678772, 0.07239888608455658, 0.05693604797124863, 0.25766414403915405, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.29880762100219727, 0.05552824214100838, 0.05443597584962845, 0.05285302922129631, 0.06568750739097595, 0.07554222643375397, 0.06968103349208832, 0.054113879799842834, 0.09120790660381317, 0.18214264512062073, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.26933014392852783, 0.06352431327104568, 0.059521228075027466, 0.052424702793359756, 0.06052157282829285, 0.12185832113027573, 0.07608654350042343, 0.09753072261810303, 0.04232607036828995, 0.05058327317237854, 0.10629311949014664, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.22074301540851593, 0.06960077583789825, 0.048723481595516205, 0.06675717234611511, 0.06954211741685867, 0.06483786553144455, 0.046523742377758026, 0.039300836622714996, 0.018016092479228973, 0.0358063168823719, 0.053022876381874084, 0.26712578535079956, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15080073475837708, 0.04732475057244301, 0.12551115453243256, 0.06240082159638405, 0.05354631692171097, 0.05578393489122391, 0.04542611539363861, 0.03888600319623947, 0.0301998108625412, 0.033370546996593475, 0.04645953327417374, 0.023437919095158577, 0.2868523597717285, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2044883519411087, 0.04972049966454506, 0.04838546738028526, 0.043853580951690674, 0.05115717276930809, 0.1007489338517189, 0.06420683115720749, 0.08302561193704605, 0.03880453109741211, 0.047957729548215866, 0.09582382440567017, 0.03239700198173523, 0.04441104456782341, 0.0950193703174591, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1623673439025879, 0.028519174084067345, 0.047843094915151596, 0.041354816406965256, 0.040284883230924606, 0.0520053468644619, 0.04226505383849144, 0.03821168839931488, 0.05986882373690605, 0.06849399209022522, 0.04734278842806816, 0.019715160131454468, 0.06202228367328644, 0.04572192579507828, 0.24398362636566162, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13128933310508728, 0.039628103375434875, 0.059519413858652115, 0.03966323286294937, 0.04406355693936348, 0.04348496347665787, 0.03726547583937645, 0.03240034729242325, 0.03720574453473091, 0.06481601297855377, 0.040474534034729004, 0.025914235040545464, 0.06507598608732224, 0.03922511264681816, 0.07744196802377701, 0.2225320190191269, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15813341736793518, 0.03911251947283745, 0.03856496512889862, 0.03587968647480011, 0.04187794774770737, 0.08161000907421112, 0.05245806276798248, 0.0681268721818924, 0.03444807231426239, 0.04342752322554588, 0.08275996893644333, 0.02933340333402157, 0.041573796421289444, 0.08478686958551407, 0.04881599545478821, 0.032352838665246964, 0.08673813939094543, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13890595734119415, 0.03169632703065872, 0.03251556307077408, 0.03538898378610611, 0.038000334054231644, 0.06974425166845322, 0.059424545615911484, 0.06768239289522171, 0.027843810617923737, 0.039054665714502335, 0.07273104041814804, 0.030109861865639687, 0.04163092002272606, 0.07565011829137802, 0.040188319981098175, 0.027483616024255753, 0.07795906811952591, 0.09399020671844482, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09691290557384491, 0.030687319114804268, 0.030817966908216476, 0.04473433643579483, 0.04171944037079811, 0.036036767065525055, 0.04101703688502312, 0.032104767858982086, 0.04344063252210617, 0.048855531960725784, 0.03457901254296303, 0.016025088727474213, 0.049941662698984146, 0.0346352756023407, 0.060656867921352386, 0.034977883100509644, 0.03454342484474182, 0.037827931344509125, 0.25048619508743286, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12059216946363449, 0.04121065139770508, 0.0509851798415184, 0.052891526371240616, 0.02886129543185234, 0.04706199839711189, 0.0292128287255764, 0.038170017302036285, 0.021046487614512444, 0.029787806794047356, 0.04406721517443657, 0.02650333382189274, 0.04127003997564316, 0.04430737346410751, 0.03080155700445175, 0.04064903408288956, 0.044745106250047684, 0.042122457176446915, 0.02844228595495224, 0.1972716897726059, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11388304829597473, 0.028255397453904152, 0.028351297602057457, 0.026958664879202843, 0.031310852617025375, 0.05977548286318779, 0.0391821525990963, 0.05072040110826492, 0.027684014290571213, 0.035564228892326355, 0.06452134251594543, 0.02426709607243538, 0.03616495057940483, 0.06838168203830719, 0.04139300435781479, 0.029094640165567398, 0.0723004937171936, 0.07349380850791931, 0.037000205367803574, 0.0352863185107708, 0.07641094923019409, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08453849703073502, 0.017416605725884438, 0.01894914172589779, 0.0165606327354908, 0.02229611948132515, 0.04174060747027397, 0.05894864723086357, 0.03893362358212471, 0.030214650556445122, 0.034642331302165985, 0.0483190156519413, 0.016358422115445137, 0.02567455545067787, 0.05309673771262169, 0.02054651454091072, 0.02046983130276203, 0.057046402245759964, 0.06604909896850586, 0.03774644806981087, 0.021413879469037056, 0.06129930540919304, 0.2077389657497406, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1124843955039978, 0.024394847452640533, 0.024523155763745308, 0.027489561587572098, 0.026104671880602837, 0.03731505200266838, 0.06748531758785248, 0.03890375792980194, 0.01798143796622753, 0.028454387560486794, 0.03756532073020935, 0.01832606829702854, 0.020088663324713707, 0.03865170478820801, 0.0232856422662735, 0.024226617068052292, 0.03960612043738365, 0.04833188280463219, 0.04500093311071396, 0.017362147569656372, 0.04009140655398369, 0.05208287015557289, 0.19024400413036346, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08183357119560242, 0.019828449934720993, 0.02687339298427105, 0.029252557083964348, 0.02518443763256073, 0.03192773088812828, 0.029049793258309364, 0.024972520768642426, 0.03350626677274704, 0.04007429629564285, 0.033517319709062576, 0.016035988926887512, 0.03204454854130745, 0.03448064625263214, 0.028517093509435654, 0.03532731533050537, 0.03583228215575218, 0.04058396443724632, 0.029941106215119362, 0.02109791524708271, 0.03671329841017723, 0.039212584495544434, 0.03543241322040558, 0.23876062035560608, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10145692527294159, 0.014237047173082829, 0.016984183341264725, 0.018636571243405342, 0.02853623777627945, 0.0365380235016346, 0.02957051247358322, 0.041983190923929214, 0.014166045933961868, 0.030081408098340034, 0.04016641527414322, 0.02007502317428589, 0.030370114371180534, 0.043534401804208755, 0.030393019318580627, 0.020717334002256393, 0.04677111282944679, 0.0549510233104229, 0.028809182345867157, 0.020043687894940376, 0.05092543363571167, 0.04084467142820358, 0.0418580137193203, 0.03848976269364357, 0.15986064076423645, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07937924563884735, 0.020674310624599457, 0.020799757912755013, 0.027988232672214508, 0.04565475508570671, 0.02656259387731552, 0.02539670839905739, 0.021243257448077202, 0.01776379905641079, 0.02520064264535904, 0.027576463297009468, 0.030607089400291443, 0.034380000084638596, 0.028568314388394356, 0.022726934403181076, 0.01605195552110672, 0.029712749645113945, 0.03276180848479271, 0.025702400133013725, 0.012878884561359882, 0.030626127496361732, 0.02036965638399124, 0.03741392120718956, 0.027850378304719925, 0.035718515515327454, 0.27639153599739075, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09605434536933899, 0.02309289574623108, 0.024199659004807472, 0.024974144995212555, 0.036829639226198196, 0.03001439943909645, 0.021069226786494255, 0.018117526546120644, 0.022569267079234123, 0.02834322676062584, 0.030505143105983734, 0.021707680076360703, 0.028211787343025208, 0.031639426946640015, 0.03916612267494202, 0.03627365827560425, 0.03311571851372719, 0.03514396771788597, 0.02574523724615574, 0.015889015048742294, 0.034675464034080505, 0.02677008882164955, 0.027143454179167747, 0.0404508002102375, 0.037603847682476044, 0.032323356717824936, 0.17837099730968475, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07476937770843506, 0.011021512560546398, 0.010706555098295212, 0.015325457789003849, 0.01990356110036373, 0.03285866603255272, 0.021936072036623955, 0.02945413999259472, 0.01642150804400444, 0.022476578131318092, 0.03722343221306801, 0.01758023351430893, 0.021954145282506943, 0.040462132543325424, 0.031415410339832306, 0.010417776182293892, 0.044297393411397934, 0.05118127167224884, 0.029565559700131416, 0.015394334681332111, 0.0490642674267292, 0.03202195093035698, 0.030905120074748993, 0.04132053256034851, 0.06578273326158524, 0.03978404775261879, 0.02049970254302025, 0.16625656187534332, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05124205723404884, 0.011390028521418571, 0.02026580646634102, 0.024529647082090378, 0.12732437252998352, 0.020083913579583168, 0.017692744731903076, 0.020381992682814598, 0.018548619002103806, 0.023206066340208054, 0.021026061847805977, 0.020521236583590508, 0.025210734456777573, 0.021994242444634438, 0.017190473154187202, 0.02080591395497322, 0.023035308346152306, 0.026458054780960083, 0.031086130067706108, 0.011454472318291664, 0.024357995018363, 0.02252495475113392, 0.024600783362984657, 0.024588001891970634, 0.02838081493973732, 0.04660612717270851, 0.043627094477415085, 0.02030046284198761, 0.21156589686870575, 0.0, 0.0, 0.0, 0.0], [0.07929293811321259, 0.023537395521998405, 0.022847289219498634, 0.01993604749441147, 0.015462876297533512, 0.025110295042395592, 0.021429050713777542, 0.015073496848344803, 0.00992409698665142, 0.015724744647741318, 0.027188707143068314, 0.020747287198901176, 0.03497635945677757, 0.029102325439453125, 0.021219033747911453, 0.01814761571586132, 0.03105819784104824, 0.03380950167775154, 0.020137565210461617, 0.012323119677603245, 0.0336926244199276, 0.02433612570166588, 0.031084517017006874, 0.03377701714634895, 0.04355444014072418, 0.032963525503873825, 0.03314930200576782, 0.02536490187048912, 0.022163692861795425, 0.2228659689426422, 0.0, 0.0, 0.0], [0.10059655457735062, 0.03096376359462738, 0.025578510016202927, 0.0202240701764822, 0.027132991701364517, 0.0294907595962286, 0.0407448373734951, 0.021403610706329346, 0.016392609104514122, 0.02252069301903248, 0.027555176988244057, 0.013599206693470478, 0.027306707575917244, 0.027806581929326057, 0.011214177124202251, 0.023586539551615715, 0.02839917503297329, 0.03066891059279442, 0.021664204075932503, 0.017032310366630554, 0.02905159257352352, 0.026549918577075005, 0.03297552466392517, 0.026790324598550797, 0.032060641795396805, 0.02729833871126175, 0.017307184636592865, 0.018768811598420143, 0.028226524591445923, 0.017085110768675804, 0.18000467121601105, 0.0, 0.0], [0.08361606299877167, 0.016581889241933823, 0.015823302790522575, 0.022005166858434677, 0.02406255342066288, 0.021028300747275352, 0.022811293601989746, 0.014412387274205685, 0.01557208877056837, 0.015104830265045166, 0.020917734131217003, 0.021798150613904, 0.02598598226904869, 0.02201320417225361, 0.022025059908628464, 0.018091218546032906, 0.02311510592699051, 0.025451799854636192, 0.02594352327287197, 0.011241885833442211, 0.024466173723340034, 0.0228698942810297, 0.031613439321517944, 0.029542673379182816, 0.02381790056824684, 0.029349016025662422, 0.025922643020749092, 0.017268173396587372, 0.0318446047604084, 0.012037734501063824, 0.027524925768375397, 0.2561413049697876, 0.0], [0.05258440971374512, 0.012508383020758629, 0.012737675569951534, 0.012144193053245544, 0.0144678158685565, 0.024812448769807816, 0.01710914634168148, 0.02516256831586361, 0.016298966482281685, 0.015160080045461655, 0.029820043593645096, 0.014290276914834976, 0.024094797670841217, 0.033742766827344894, 0.021632254123687744, 0.018924105912446976, 0.03819216787815094, 0.04115181416273117, 0.024165764451026917, 0.020635638386011124, 0.044185325503349304, 0.03793709725141525, 0.029636353254318237, 0.03152839466929436, 0.04542306438088417, 0.035963598638772964, 0.030482539907097816, 0.04609914869070053, 0.03206766024231911, 0.01766294240951538, 0.03169601783156395, 0.04462910816073418, 0.10305341333150864]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7707713842391968, 0.22922858595848083, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7813623547554016, 0.109785296022892, 0.10885230451822281, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.535038948059082, 0.11194943636655807, 0.15013694763183594, 0.20287470519542694, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.41803574562072754, 0.1603638231754303, 0.1074029952287674, 0.10625781118869781, 0.20793959498405457, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4349866509437561, 0.09579858183860779, 0.08251801878213882, 0.10028504580259323, 0.15332908928394318, 0.1330825835466385, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.44621357321739197, 0.05624214559793472, 0.05040299892425537, 0.07084295898675919, 0.09360906481742859, 0.11472620815038681, 0.16796305775642395, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3609195351600647, 0.07294464111328125, 0.05191005766391754, 0.07159046083688736, 0.09321668744087219, 0.09543665498495102, 0.14539918303489685, 0.10858277231454849, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3171406686306, 0.07683130353689194, 0.05552283674478531, 0.06445366144180298, 0.08586590737104416, 0.10643255710601807, 0.1452113687992096, 0.09439337253570557, 0.05414833128452301, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19288603961467743, 0.1134386956691742, 0.04530505836009979, 0.08361203223466873, 0.05763273686170578, 0.11914196610450745, 0.11441227793693542, 0.12364513427019119, 0.07640958577394485, 0.07351639866828918, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.25174087285995483, 0.060194116085767746, 0.04930208995938301, 0.059223245829343796, 0.08453336358070374, 0.07307681441307068, 0.12308554351329803, 0.08293969184160233, 0.06380272656679153, 0.07185835391283035, 0.08024313300848007, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2479359656572342, 0.048557743430137634, 0.061716966331005096, 0.06616783887147903, 0.0878860130906105, 0.05920383334159851, 0.09128588438034058, 0.06866813451051712, 0.060628652572631836, 0.08399352431297302, 0.0656941756606102, 0.05826116353273392, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15039603412151337, 0.0707230269908905, 0.036188021302223206, 0.03812884911894798, 0.05496574938297272, 0.09552797675132751, 0.1278020143508911, 0.09101750701665878, 0.04133947193622589, 0.06619426608085632, 0.11493087559938431, 0.03526817262172699, 0.0775180533528328, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.21730750799179077, 0.052248772233724594, 0.04279686510562897, 0.05130988731980324, 0.07021071761846542, 0.06084864214062691, 0.10094168037176132, 0.06729137152433395, 0.05347270891070366, 0.05803239718079567, 0.063866026699543, 0.0414368212223053, 0.05429811403155327, 0.06593842804431915, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14158985018730164, 0.08735613524913788, 0.041096266359090805, 0.07771317660808563, 0.0664159283041954, 0.06084612384438515, 0.07890869677066803, 0.06467185169458389, 0.041567541658878326, 0.05314023420214653, 0.060331959277391434, 0.049903471022844315, 0.06147686019539833, 0.06152055785059929, 0.053461313247680664, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11277392506599426, 0.05140271782875061, 0.03480074182152748, 0.044109929352998734, 0.04927106201648712, 0.08066023141145706, 0.09371130168437958, 0.07634277641773224, 0.03460073471069336, 0.0555649995803833, 0.09000370651483536, 0.027950923889875412, 0.058086883276700974, 0.09759874641895294, 0.058530546724796295, 0.03459078073501587, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19136789441108704, 0.04588800296187401, 0.037184182554483414, 0.04478847235441208, 0.058455415070056915, 0.0516231432557106, 0.08451797068119049, 0.056617289781570435, 0.04477924853563309, 0.04681586101651192, 0.05177522450685501, 0.034743115305900574, 0.04365590214729309, 0.05257752537727356, 0.045719120651483536, 0.05523938685655594, 0.05425224080681801, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.178098663687706, 0.03651316463947296, 0.035378020256757736, 0.04660060256719589, 0.056067463010549545, 0.05542533099651337, 0.06580902636051178, 0.058695923537015915, 0.04460207000374794, 0.03845333680510521, 0.05522071197628975, 0.02812928520143032, 0.038876280188560486, 0.056352097541093826, 0.04307202994823456, 0.048592109233140945, 0.05801301822066307, 0.056100811809301376, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11277126520872116, 0.04416341334581375, 0.01679084822535515, 0.03922899439930916, 0.030570238828659058, 0.08429381251335144, 0.05829833820462227, 0.07985574752092361, 0.027740228921175003, 0.024607932195067406, 0.0856802761554718, 0.01213553361594677, 0.021223612129688263, 0.08979952335357666, 0.02439562790095806, 0.018000086769461632, 0.0979786366224289, 0.08564859628677368, 0.04681731015443802, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15648671984672546, 0.043593376874923706, 0.029469136148691177, 0.02338278852403164, 0.03989394009113312, 0.06601448357105255, 0.07239893823862076, 0.049598775804042816, 0.025720087811350822, 0.028729889541864395, 0.07268749177455902, 0.020104380324482918, 0.038949914276599884, 0.0779348835349083, 0.031850170344114304, 0.029572613537311554, 0.08162925392389297, 0.04983029142022133, 0.04757083207368851, 0.014582027681171894, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.16675154864788055, 0.040216755121946335, 0.031946271657943726, 0.03915555030107498, 0.04867977276444435, 0.043519362807273865, 0.07027734071016312, 0.04676978662610054, 0.03737045079469681, 0.03785951808094978, 0.04185349866747856, 0.029438791796565056, 0.03496702387928963, 0.04187152162194252, 0.036659594625234604, 0.04418949410319328, 0.04244700446724892, 0.04308999702334404, 0.055408261716365814, 0.02402997761964798, 0.04349849000573158, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.21311703324317932, 0.03452147915959358, 0.026512037962675095, 0.037284668534994125, 0.04200230911374092, 0.05378075689077377, 0.06410719454288483, 0.040195900946855545, 0.029513953253626823, 0.030562065541744232, 0.048140499740839005, 0.019645733758807182, 0.023855963721871376, 0.04712359979748726, 0.027805231511592865, 0.025632821023464203, 0.04690323397517204, 0.042114999145269394, 0.04245338588953018, 0.016209973022341728, 0.04720157012343407, 0.041315577924251556, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.16099101305007935, 0.03149211406707764, 0.025505999103188515, 0.03203628212213516, 0.038119371980428696, 0.05386463925242424, 0.05289200320839882, 0.06297367811203003, 0.03872942551970482, 0.03070300817489624, 0.04861500486731529, 0.016964146867394447, 0.020984871312975883, 0.04743487387895584, 0.026178555563092232, 0.032817039638757706, 0.048741020262241364, 0.03702303394675255, 0.05028446391224861, 0.014425938948988914, 0.04926920309662819, 0.02748514525592327, 0.052469104528427124, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13924191892147064, 0.04732196778059006, 0.022920068353414536, 0.037826746702194214, 0.03733159974217415, 0.056281980127096176, 0.06773107498884201, 0.0638374611735344, 0.028824560344219208, 0.028368612751364708, 0.05008407309651375, 0.014775178395211697, 0.021058905869722366, 0.04819713160395622, 0.017685124650597572, 0.01545622292906046, 0.0484280101954937, 0.037099532783031464, 0.042473260313272476, 0.01842682436108589, 0.04861331358551979, 0.022673960775136948, 0.05998954549431801, 0.02535295858979225, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.195623517036438, 0.03233327716588974, 0.022874081507325172, 0.038390763103961945, 0.03449878469109535, 0.04616905376315117, 0.055537495762109756, 0.03960668295621872, 0.02309839427471161, 0.024249285459518433, 0.04043305665254593, 0.016078554093837738, 0.021624945104122162, 0.03903055936098099, 0.026520881801843643, 0.028113437816500664, 0.03823632746934891, 0.035426102578639984, 0.04174575209617615, 0.02109713852405548, 0.03767955303192139, 0.037286918610334396, 0.04423920437693596, 0.022921495139598846, 0.037184685468673706, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0581965334713459, 0.038897398859262466, 0.02600524015724659, 0.02356790192425251, 0.022979989647865295, 0.053841520100831985, 0.058061059564352036, 0.06858771294355392, 0.037459827959537506, 0.029159093275666237, 0.05474674701690674, 0.013592950068414211, 0.027647046372294426, 0.05575403571128845, 0.030779868364334106, 0.01613735221326351, 0.05745428055524826, 0.03968728333711624, 0.03406553715467453, 0.013110978528857231, 0.059776563197374344, 0.016043761745095253, 0.029699290171265602, 0.020566539838910103, 0.08347290754318237, 0.03070860728621483, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15561480820178986, 0.030323194339871407, 0.02106756716966629, 0.038600414991378784, 0.025089262053370476, 0.04719521105289459, 0.04255649819970131, 0.051662590354681015, 0.030368102714419365, 0.020133620128035545, 0.04255696386098862, 0.018511781468987465, 0.01867171935737133, 0.0414302833378315, 0.022022131830453873, 0.02106102742254734, 0.0416661836206913, 0.027249205857515335, 0.04048159345984459, 0.015536420047283173, 0.04219621419906616, 0.017007341608405113, 0.06220947951078415, 0.015703151002526283, 0.05443187430500984, 0.04092409461736679, 0.015729239210486412, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1521887332201004, 0.03232048079371452, 0.026277603581547737, 0.03215136379003525, 0.03708053380250931, 0.03843334689736366, 0.042185209691524506, 0.034409213811159134, 0.029296185821294785, 0.02475224994122982, 0.03461232781410217, 0.021795086562633514, 0.023419681936502457, 0.03386644273996353, 0.028580598533153534, 0.03440327197313309, 0.033156976103782654, 0.026710400357842445, 0.04213565215468407, 0.019021350890398026, 0.032785814255476, 0.02683998830616474, 0.03304455429315567, 0.02330264076590538, 0.03390035033226013, 0.03944924846291542, 0.021334253251552582, 0.042546432465314865, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10210391879081726, 0.048305340111255646, 0.027957914397120476, 0.028134863823652267, 0.04368893802165985, 0.03765960410237312, 0.04012182354927063, 0.03589581325650215, 0.024491794407367706, 0.019768042489886284, 0.03467409312725067, 0.015431461855769157, 0.032616548240184784, 0.03414339944720268, 0.027461368590593338, 0.020323220640420914, 0.03445841744542122, 0.015827562659978867, 0.030705846846103668, 0.020360933616757393, 0.03558044880628586, 0.034790296107530594, 0.03518148139119148, 0.028572604060173035, 0.04333385452628136, 0.03896389529109001, 0.017587700858712196, 0.04553375393152237, 0.04632510989904404, 0.0, 0.0, 0.0, 0.0], [0.1184174120426178, 0.026689782738685608, 0.02854764647781849, 0.03516455739736557, 0.03519093617796898, 0.030704878270626068, 0.042546890676021576, 0.023103507235646248, 0.025463124737143517, 0.029678549617528915, 0.028395220637321472, 0.019178256392478943, 0.02383357845246792, 0.027640020474791527, 0.03381752222776413, 0.0396401509642601, 0.027574418112635612, 0.021856164559721947, 0.03966765105724335, 0.03941834345459938, 0.02711562067270279, 0.030498063191771507, 0.034932829439640045, 0.0323205292224884, 0.027840541675686836, 0.03877578675746918, 0.021124795079231262, 0.03861142694950104, 0.032771822065114975, 0.019479982554912567, 0.0, 0.0, 0.0], [0.05433792620897293, 0.019247835502028465, 0.011686461977660656, 0.014118609949946404, 0.01390978042036295, 0.05736807361245155, 0.0345463827252388, 0.046650584787130356, 0.013026447966694832, 0.013271295465528965, 0.05952775105834007, 0.009063147008419037, 0.017364196479320526, 0.061355650424957275, 0.01263465266674757, 0.015145537443459034, 0.06425532698631287, 0.046270616352558136, 0.01955636776983738, 0.012543462216854095, 0.06913558393716812, 0.034823667258024216, 0.030005954205989838, 0.012261823751032352, 0.07985591888427734, 0.018996428698301315, 0.010722929611802101, 0.0865454226732254, 0.017174094915390015, 0.022144164890050888, 0.022453978657722473, 0.0, 0.0], [0.05849646031856537, 0.03437737002968788, 0.013603095896542072, 0.02417043223977089, 0.01900586113333702, 0.0654640942811966, 0.030622966587543488, 0.03573206812143326, 0.01885945349931717, 0.012241484597325325, 0.06282190978527069, 0.011835752986371517, 0.009552359580993652, 0.06431794911623001, 0.02158958464860916, 0.010912974365055561, 0.0666591003537178, 0.02399745211005211, 0.026596758514642715, 0.006153556052595377, 0.06984329968690872, 0.03723869100213051, 0.030046707019209862, 0.021686814725399017, 0.043640609830617905, 0.021209606900811195, 0.013490457087755203, 0.04632722586393356, 0.020434511825442314, 0.013362264260649681, 0.018568990752100945, 0.04714024439454079, 0.0], [0.1352868676185608, 0.02973337098956108, 0.02611132338643074, 0.03753575682640076, 0.038881585001945496, 0.0331667996942997, 0.04824390634894371, 0.03046831116080284, 0.026038208976387978, 0.024123696610331535, 0.027892732992768288, 0.02270994521677494, 0.024185286834836006, 0.02611454948782921, 0.02320966310799122, 0.025175290182232857, 0.024614792317152023, 0.02112315408885479, 0.02968505024909973, 0.01816747523844242, 0.023064401000738144, 0.02059807814657688, 0.034411393105983734, 0.021083395928144455, 0.02253883145749569, 0.0322815477848053, 0.013987251557409763, 0.02414492331445217, 0.02700231224298477, 0.012929446063935757, 0.03616539016366005, 0.02684023790061474, 0.03248502314090729]]]}
     )
@@ -762,7 +762,7 @@ <h3>Activation Patching on the Indirect Object Identification Task<a class="head
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "ddd91d21a45e49a8a90fd6693fc0fe90"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "bb02e41e51664a96b3c06abd21683006"}</script></div>
 </div>
 <p>We can now visualize the results, and see that this computation is extremely localised within the model. Initially, the second subject (Mary) token is all that matters (naturally, as it’s the only different token), and all relevant information remains here until heads in layer 7 and 8 move this to the final token where it’s used to predict the indirect object. (Note - the heads are in layer 7 and 8, not 8 and 9, because we patched in the residual stream at the <em>start</em> of each layer)</p>
 <div class="nbinput docutils container">
@@ -783,9 +783,9 @@ <h3>Activation Patching on the Indirect Object Identification Task<a class="head
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="a86e9b27-7dd3-47fb-9141-10fa1ea8195f" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("a86e9b27-7dd3-47fb-9141-10fa1ea8195f")) {                    Plotly.newPlot(                        "a86e9b27-7dd3-47fb-9141-10fa1ea8195f",                        [{"coloraxis":"coloraxis","name":"0","x":["\u003c|endoftext|\u003e_0","After_1"," John_2"," and_3"," Mary_4"," went_5"," to_6"," the_7"," store_8",",_9"," Mary_10"," gave_11"," a_12"," bottle_13"," of_14"," milk_15"," to_16"],"z":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9981481432914734,0.001600604155100882,0.0001503795647295192,-0.0003718698862940073,-2.257052983622998e-05,-0.000628303736448288,-0.000515179184731096],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9980564713478088,0.002283838577568531,0.00018205988453701138,-0.0005045737489126623,-0.0002681270125322044,-5.153151141712442e-05,-0.0012816254748031497],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9967373609542847,0.004081866703927517,0.0009732521721161902,4.378139055916108e-05,-0.00015962531324476004,-0.00033638247987255454,-0.0019441929180175066],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9905908107757568,0.019987020641565323,0.0018955166451632977,0.0010141782695427537,-6.75756236887537e-05,0.000911251176148653,-0.0019012272823601961],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9616515636444092,0.08534706383943558,0.0052040028385818005,0.003052052343264222,0.00019688029715325683,0.0011058200616389513,-0.002284654416143894],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.963099479675293,0.0843719094991684,0.00412211287766695,0.0007176341023296118,0.00010319898865418509,0.0010020771296694875,-0.004215250723063946],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9359182119369507,0.11111730337142944,0.007705116644501686,0.0003754050412680954,0.0003647996054496616,0.0013266304740682244,0.01874428242444992],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.7701544761657715,0.03741976246237755,0.0020680588204413652,-8.321183850057423e-05,0.00013447141100186855,0.0017246060306206346,0.44990694522857666],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0965060144662857,0.025925789028406143,0.0019711144268512726,0.00032904025283642113,0.00042367333662696183,0.00188477523624897,0.8994728922843933],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.023322155699133873,0.018537340685725212,0.001587143400683999,0.000526600400917232,0.0002531706413719803,0.0008727724780328572,0.9612765908241272],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.008559261448681355,0.006339463870972395,0.0005807152483612299,-0.000342636980349198,0.00010958944039884955,0.0006486988277174532,0.9495824575424194]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Position: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Position"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Normalized Logit Difference After Patching Residual Stream on the IOI Task"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="2edcf647-342a-478a-992e-622f7caf82dc" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("2edcf647-342a-478a-992e-622f7caf82dc")) {                    Plotly.newPlot(                        "2edcf647-342a-478a-992e-622f7caf82dc",                        [{"coloraxis":"coloraxis","name":"0","x":["\u003c|endoftext|\u003e_0","After_1"," John_2"," and_3"," Mary_4"," went_5"," to_6"," the_7"," store_8",",_9"," Mary_10"," gave_11"," a_12"," bottle_13"," of_14"," milk_15"," to_16"],"z":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9981481432914734,0.001600604155100882,0.0001503795647295192,-0.0003718698862940073,-2.257052983622998e-05,-0.000628303736448288,-0.000515179184731096],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9980564713478088,0.002283838577568531,0.00018205988453701138,-0.0005045737489126623,-0.0002681270125322044,-5.153151141712442e-05,-0.0012816254748031497],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9967373609542847,0.004081866703927517,0.0009732521721161902,4.378139055916108e-05,-0.00015962531324476004,-0.00033638247987255454,-0.0019441929180175066],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9905908107757568,0.019987020641565323,0.0018955166451632977,0.0010141782695427537,-6.75756236887537e-05,0.000911251176148653,-0.0019012272823601961],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9616515636444092,0.08534706383943558,0.0052040028385818005,0.003052052343264222,0.00019688029715325683,0.0011058200616389513,-0.002284654416143894],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.963099479675293,0.0843719094991684,0.00412211287766695,0.0007176341023296118,0.00010319898865418509,0.0010020771296694875,-0.004215250723063946],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9359182119369507,0.11111730337142944,0.007705116644501686,0.0003754050412680954,0.0003647996054496616,0.0013266304740682244,0.01874428242444992],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.7701544761657715,0.03741976246237755,0.0020680588204413652,-8.321183850057423e-05,0.00013447141100186855,0.0017246060306206346,0.44990694522857666],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0965060144662857,0.025925789028406143,0.0019711144268512726,0.00032904025283642113,0.00042367333662696183,0.00188477523624897,0.8994728922843933],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.023322155699133873,0.018537340685725212,0.001587143400683999,0.000526600400917232,0.0002531706413719803,0.0008727724780328572,0.9612765908241272],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.008559261448681355,0.006339463870972395,0.0005807152483612299,-0.000342636980349198,0.00010958944039884955,0.0006486988277174532,0.9495824575424194]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Position: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Position"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Normalized Logit Difference After Patching Residual Stream on the IOI Task"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('a86e9b27-7dd3-47fb-9141-10fa1ea8195f');
+var gd = document.getElementById('2edcf647-342a-478a-992e-622f7caf82dc');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -847,9 +847,9 @@ <h2>Hooks: Accessing Activations<a class="headerlink" href="#Hooks:-Accessing-Ac
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="d81bb9d3-dd8a-47ab-a254-a6ee2bc9adcc" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("d81bb9d3-dd8a-47ab-a254-a6ee2bc9adcc")) {                    Plotly.newPlot(                        "d81bb9d3-dd8a-47ab-a254-a6ee2bc9adcc",                        [{"hovertemplate":"variable=0\u003cbr\u003eindex=%{x}\u003cbr\u003evalue=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"0","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines","name":"0","orientation":"v","showlegend":true,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98],"xaxis":"x","y":[11.934269905090332,14.23181438446045,11.729150772094727,15.214762687683105,13.592933654785156,14.211908340454102,13.205050468444824,13.07355785369873,11.535038948059082,12.773419380187988,13.354753494262695,12.027746200561523,12.879249572753906,11.845925331115723,12.831506729125977,11.792170524597168,12.737360000610352,12.617140769958496,12.619315147399902,12.312422752380371,11.479436874389648,12.757272720336914,11.545267105102539,12.173948287963867,11.990314483642578,11.905129432678223,11.678251266479492,11.099139213562012,10.786980628967285,12.338418960571289,11.62226676940918,11.612360000610352,10.657169342041016,11.615835189819336,11.122636795043945,10.654256820678711,11.861252784729004,11.142145156860352,11.718962669372559,11.342132568359375,11.637442588806152,11.20296859741211,12.066685676574707,10.65315055847168,11.302213668823242,11.483949661254883,10.655682563781738,11.83389949798584,11.766695976257324,11.974489212036133,3.564096450805664,1.2277376651763916,0.9820038676261902,1.200751543045044,0.4091821312904358,0.5843209624290466,0.3591119647026062,0.43036898970603943,0.24223056435585022,0.44049954414367676,0.41071295738220215,0.21330496668815613,0.15890493988990784,0.31460171937942505,0.08366963267326355,0.2029363214969635,0.17958413064479828,0.1480543464422226,0.034813292324543,0.06144014745950699,0.09302718937397003,0.06429728120565414,0.4040629267692566,0.12488675117492676,0.08299500495195389,0.02960968390107155,0.01776202768087387,0.03346758708357811,0.10293184220790863,0.0828002542257309,0.008384309709072113,0.031925592571496964,0.1624717116355896,0.05249609425663948,0.02139073610305786,0.03643417730927467,0.07777489721775055,0.1070786863565445,0.03805286064743996,0.11206188052892685,0.1607268750667572,0.13429032266139984,0.16408373415470123,0.05713384598493576,0.12058018147945404,0.09626038372516632,0.04750080779194832,0.1064792275428772,0.15790846943855286],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"index"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"value"}},"legend":{"title":{"text":"variable"},"tracegroupgap":0},"title":{"text":"Loss by position on random repeated tokens"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="9fc76eb5-5519-441b-804f-b9d3f284b6b4" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("9fc76eb5-5519-441b-804f-b9d3f284b6b4")) {                    Plotly.newPlot(                        "9fc76eb5-5519-441b-804f-b9d3f284b6b4",                        [{"hovertemplate":"variable=0\u003cbr\u003eindex=%{x}\u003cbr\u003evalue=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"0","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines","name":"0","orientation":"v","showlegend":true,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98],"xaxis":"x","y":[11.293755531311035,13.661523818969727,14.26708984375,13.05138874053955,14.200154304504395,13.794110298156738,13.165082931518555,12.725430488586426,12.198771476745605,13.159016609191895,11.78315258026123,11.827463150024414,12.019637107849121,12.807088851928711,12.95734691619873,12.7166109085083,11.805150032043457,11.203890800476074,13.748982429504395,11.939554214477539,12.111616134643555,11.838865280151367,11.640402793884277,11.894838333129883,11.969301223754883,10.904563903808594,11.69995403289795,11.886842727661133,11.184012413024902,11.828400611877441,11.488662719726562,11.595769882202148,11.27686595916748,12.046403884887695,11.61715030670166,11.7785062789917,12.143930435180664,11.37479019165039,11.056105613708496,11.812862396240234,11.769815444946289,11.787055969238281,11.405266761779785,11.02073860168457,12.023134231567383,11.26836109161377,10.82592487335205,11.290939331054688,11.096704483032227,11.10577392578125,3.174548387527466,1.6711616516113281,0.613044261932373,0.33394116163253784,0.4114641547203064,0.5941733717918396,0.30370816588401794,0.14971032738685608,0.16561320424079895,0.1323113888502121,0.19412001967430115,0.2268151342868805,0.08987182378768921,0.03984134644269943,0.09262757748365402,0.1882668137550354,0.16898849606513977,0.15458349883556366,0.09763779491186142,0.12957245111465454,0.2606273293495178,0.03914522007107735,0.049064647406339645,0.0566009059548378,0.012886052019894123,0.031596552580595016,0.04321982339024544,0.07121316343545914,0.0856885313987732,0.023817967623472214,0.048696257174015045,0.0647251158952713,0.07937098294496536,0.1875138133764267,0.05469178408384323,0.04853184148669243,0.04821369796991348,0.10546024888753891,0.16193492710590363,0.02764531411230564,0.1413104087114334,0.04153662547469139,0.11765837669372559,0.04498446732759476,0.2020321786403656,0.06594666093587875,0.06909110397100449,0.08714185655117035,0.10666583478450775],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"index"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"value"}},"legend":{"title":{"text":"variable"},"tracegroupgap":0},"title":{"text":"Loss by position on random repeated tokens"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('d81bb9d3-dd8a-47ab-a254-a6ee2bc9adcc');
+var gd = document.getElementById('9fc76eb5-5519-441b-804f-b9d3f284b6b4');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -929,9 +929,9 @@ <h2>Hooks: Accessing Activations<a class="headerlink" href="#Hooks:-Accessing-Ac
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="b2a3ed8a-ba81-4aac-a1a1-60e16774f655" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("b2a3ed8a-ba81-4aac-a1a1-60e16774f655")) {                    Plotly.newPlot(                        "b2a3ed8a-ba81-4aac-a1a1-60e16774f655",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.010606949217617512,0.00013902947830501944,0.01081207673996687,5.408500101111713e-07,0.00022116937907412648,7.771956006763503e-05,0.00958997942507267,0.0009253817261196673,0.009193798527121544,0.009053227491676807,0.006911635398864746,0.014761571772396564],[0.0008132570656016469,0.0004268861666787416,0.002124645747244358,0.01425299420952797,0.005162382964044809,0.010268471203744411,0.015934234485030174,0.013339280150830746,0.012906215153634548,0.016355415806174278,0.006486278492957354,0.0005485203000716865],[0.004910458344966173,0.019308894872665405,0.0012424595188349485,0.0010258526308462024,0.013039778918027878,0.0015818236861377954,0.00429101288318634,0.008304944261908531,0.003105038544163108,0.000971563218627125,0.0007218025857582688,0.010301434434950352],[0.015241926535964012,0.006064379122108221,0.00234038638882339,0.01002125907689333,0.021940935403108597,0.01423431932926178,0.0017911610193550587,0.0017028121510520577,0.005022364668548107,0.012421459890902042,0.00839681550860405,0.005942116491496563],[0.015156026929616928,0.013609741814434528,0.01541218627244234,0.00972918700426817,0.018237268552184105,0.013402285054326057,0.009189526550471783,0.0018425184534862638,0.015492293052375317,0.013836968690156937,0.0181787870824337,3.1250738885546525e-08],[0.42406418919563293,0.9055006504058838,0.015690578147768974,0.01017544511705637,0.01198639441281557,0.9278296828269958,0.012438608333468437,0.018412206321954727,0.027197834104299545,0.028249241411685944,0.019754743203520775,0.018243789672851562],[0.012461011298000813,0.01634187251329422,0.019040238112211227,0.015691598877310753,0.02333749644458294,0.012633594684302807,0.029116196557879448,0.012918342836201191,0.009929743595421314,0.92023104429245,0.035106346011161804,0.015322730876505375],[0.011738991364836693,0.17485056817531586,0.8415674567222595,0.01858351193368435,0.018508559092879295,0.018142390996217728,0.04514370858669281,0.08067239820957184,0.017961328849196434,0.019256191328167915,0.9187174439430237,0.052430037409067154],[0.01744176633656025,0.3839356303215027,0.01593034528195858,0.0451587438583374,0.018434638157486916,0.01456737332046032,0.14198516309261322,0.014199082739651203,0.03386015444993973,0.0334649458527565,0.06564274430274963,0.02554652839899063],[0.2423967868089676,0.19136165082454681,0.1043233796954155,0.012038059532642365,0.0845528319478035,0.027191946282982826,0.4313036799430847,0.028380122035741806,0.04645952582359314,0.4677841365337372,0.017513224855065346,0.03886174410581589],[0.3296402394771576,0.48611578345298767,0.0376255176961422,0.14191953837871552,0.06014999374747276,0.01589331030845642,0.28858682513237,0.479020357131958,0.05016819015145302,0.015279495157301426,0.14848238229751587,0.25642627477645874],[0.01643357053399086,0.05329973250627518,0.035110954195261,0.011042965576052666,0.03575966879725456,0.10564529150724411,0.048889338970184326,0.06571993976831436,0.009506475180387497,0.30028679966926575,0.3763805329799652,0.025962304323911667]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Induction Score by Head"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="ecf684dd-de93-401f-8da4-35a69a99f28f" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("ecf684dd-de93-401f-8da4-35a69a99f28f")) {                    Plotly.newPlot(                        "ecf684dd-de93-401f-8da4-35a69a99f28f",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.010750330053269863,0.00012081558088539168,0.010657578706741333,7.758595756968134e-07,0.00018270811415277421,0.0001077339329640381,0.009928593412041664,0.0006260719965212047,0.009389315731823444,0.009186347015202045,0.00702526094391942,0.014964522793889046],[0.0007892780122347176,0.0003822337312158197,0.0021375513169914484,0.014263155870139599,0.005154898855835199,0.010422330349683762,0.015344158746302128,0.012839337810873985,0.013111173175275326,0.01605435460805893,0.006542571354657412,0.0005330032436177135],[0.004155242349952459,0.017997099086642265,0.0025227037258446217,0.0013244586298242211,0.014491848647594452,0.002136210212484002,0.003679797286167741,0.008564706891775131,0.003503501648083329,0.001399003784172237,0.0006941384635865688,0.010122609324753284],[0.015284194611012936,0.007457489613443613,0.0030496998224407434,0.01488673035055399,0.02102597802877426,0.011635568924248219,0.002769281156361103,0.0009841499850153923,0.005578458774834871,0.014012619853019714,0.008734799921512604,0.007212471682578325],[0.016826970502734184,0.01410447433590889,0.014252868480980396,0.008973448537290096,0.018241047859191895,0.014347823336720467,0.008000489324331284,0.002079280326142907,0.015254753641784191,0.01450068037956953,0.018436405807733536,8.373506332759462e-10],[0.4491550922393799,0.9050912857055664,0.015386047773063183,0.00875406339764595,0.01097724586725235,0.9397631883621216,0.011584484949707985,0.01816929504275322,0.028745127841830254,0.027196679264307022,0.017204662784934044,0.017268218100070953],[0.010569052770733833,0.016979962587356567,0.01757051981985569,0.013518791645765305,0.021789412945508957,0.01171425823122263,0.02984258159995079,0.010995764285326004,0.00976125244051218,0.9289324283599854,0.03460867702960968,0.014708924107253551],[0.014089317992329597,0.18321795761585236,0.850297749042511,0.018563423305749893,0.0190876517444849,0.01761588081717491,0.04573224112391472,0.08691924065351486,0.01809566281735897,0.01936078630387783,0.9231538772583008,0.05635310709476471],[0.01650477573275566,0.38255754113197327,0.01667136326432228,0.04383406788110733,0.01883339323103428,0.01519932970404625,0.14662882685661316,0.012798622250556946,0.02996883913874626,0.03196403756737709,0.06725890934467316,0.019399145618081093],[0.2549801766872406,0.17897091805934906,0.11086569726467133,0.013412022031843662,0.09373626857995987,0.02757657691836357,0.43764016032218933,0.02954663708806038,0.05084982514381409,0.4542059898376465,0.018278131261467934,0.044380251318216324],[0.35109150409698486,0.4873502254486084,0.04331290349364281,0.1357107311487198,0.06467364728450775,0.016985448077321053,0.3039875626564026,0.46872273087501526,0.050012968480587006,0.01698106899857521,0.16451457142829895,0.2619016468524933],[0.017976440489292145,0.0546535849571228,0.03751886636018753,0.011606932617723942,0.0373089537024498,0.10333973169326782,0.054286956787109375,0.06869052350521088,0.009068110026419163,0.3063192069530487,0.4115610718727112,0.023157836869359016]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Induction Score by Head"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('b2a3ed8a-ba81-4aac-a1a1-60e16774f655');
+var gd = document.getElementById('ecf684dd-de93-401f-8da4-35a69a99f28f');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -999,11 +999,11 @@ <h2>Hooks: Accessing Activations<a class="headerlink" href="#Hooks:-Accessing-Ac
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div id="circuits-vis-61a73e43-0cbc" style="margin: 15px 0;"/>
+<div id="circuits-vis-fe1e1e85-ccb2" style="margin: 15px 0;"/>
     <script crossorigin type="module">
     import { render, AttentionPatterns } from "https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js";
     render(
-      "circuits-vis-61a73e43-0cbc",
+      "circuits-vis-fe1e1e85-ccb2",
       AttentionPatterns,
       {"tokens": ["use", " advice", " Social", "\u00f6", "\u00b7", " fought", " Le", " allegedly", " NO", "alth", "car", " prepared", "new", "rant", "roll", " hours", " published", "66", "ension", " 44", "use", " advice", " Social", "\u00f6", "\u00b7", " fought", " Le", " allegedly", " NO", "alth", "car", " prepared", "new", "rant", "roll", " hours", " published", "66", "ension", " 44"], "attention": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9737270474433899, 0.02627299167215824, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9820428490638733, 0.017020218074321747, 0.0009368443279527128, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9895542860031128, 0.008665801025927067, 0.00041197543032467365, 0.0013679895782843232, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8543054461479187, 0.07801821082830429, 0.0008415375486947596, 0.0001359905581921339, 0.06669880449771881, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.937433660030365, 0.03300206735730171, 0.0015577399171888828, 2.535264457037556e-06, 0.0010925547685474157, 0.026911458000540733, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9769211411476135, 0.0038436795584857464, 2.234023304481525e-05, 3.5218730772612616e-05, 0.00518348254263401, 0.012176254764199257, 0.0018179015023633838, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9473504424095154, 0.013174979016184807, 0.0013492192374542356, 1.1802491826529149e-05, 0.0009449502103962004, 0.01131904125213623, 0.018021006137132645, 0.00782855786383152, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9847128391265869, 0.0010781444143503904, 0.0021734496112912893, 5.4822653510200325e-06, 0.0004914247547276318, 0.0013570826267823577, 0.00010185814608121291, 0.0002853855839930475, 0.009794436395168304, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9915198683738708, 0.004483341239392757, 0.00012727164721582085, 0.00016702155699022114, 0.0016301742289215326, 0.0011521608103066683, 0.00032312856637872756, 0.00012646272080019116, 0.00039313736488111317, 7.735046528978273e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8908806443214417, 0.024311941117048264, 1.7341310012852773e-05, 4.157719376962632e-05, 0.0008967609610408545, 0.07334917038679123, 0.0009482790483161807, 0.004280843771994114, 0.00516867870464921, 7.830337381165009e-06, 9.693232277641073e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8170808553695679, 0.1351710557937622, 0.011989916674792767, 1.1421708222769666e-05, 0.0003511959221214056, 0.009450665675103664, 0.01946333795785904, 0.0006557459710165858, 0.0005760999629274011, 2.9927030482213013e-05, 1.6589872757322155e-05, 0.00520310876891017, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9082695841789246, 0.006068154238164425, 0.013871855102479458, 0.0008237077272497118, 0.01190832071006298, 0.015542060136795044, 0.008354832418262959, 0.002078164601698518, 0.0013173240004107356, 0.0021398039534687996, 0.0039441632106900215, 0.0012376613449305296, 0.024444298818707466, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9479591250419617, 0.0021026728209108114, 0.011938490904867649, 0.00012338497617747635, 3.5374921480979538e-06, 0.00014498813834507018, 0.0005875456845387816, 2.5534465748933144e-05, 0.0013608969748020172, 0.0003395687963347882, 0.010076268576085567, 0.015790557488799095, 0.006346334703266621, 0.0032010283321142197, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9393549561500549, 0.006392320618033409, 0.0018427377799525857, 6.1162154452176765e-06, 0.00033587400685064495, 0.002051525516435504, 0.003801509505137801, 0.0012357983505353332, 0.00021948102221358567, 0.0003869360953103751, 5.012243491364643e-05, 0.008153212256729603, 0.026924701407551765, 0.002937993500381708, 0.0063067772425711155, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9339622259140015, 0.0017828107811510563, 0.005864636041224003, 0.0001995021157199517, 7.227134483400732e-05, 0.0014535371446982026, 0.0025924311485141516, 0.00048593789688311517, 0.002229832811281085, 0.00015120484749786556, 0.012292915023863316, 0.005057850386947393, 0.01236859429627657, 0.0039444840513169765, 0.006275098770856857, 0.011266663670539856, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8931334018707275, 0.0015468214405700564, 0.013001682236790657, 7.966378689161502e-06, 5.864375998498872e-05, 0.0008863414404913783, 0.00320208678022027, 3.214761454728432e-05, 0.00018022459698840976, 1.1455733329057693e-05, 7.600126264151186e-05, 0.00042027299059554935, 0.0016126197297126055, 0.028539275750517845, 0.01053555216640234, 0.025432037189602852, 0.021323570981621742, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9847024083137512, 0.000458244961919263, 0.00017220497829839587, 6.160992711556901e-07, 4.782819814863615e-06, 0.0005806140834465623, 0.0004461870121303946, 0.0004120134108234197, 0.0013038699980825186, 0.0003176067257300019, 6.994167779339477e-05, 0.0013941116631031036, 5.583090751315467e-05, 0.0009110373794101179, 0.00019558057829272002, 0.0003960288013331592, 0.0011691722320392728, 0.007409730460494757, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8922597765922546, 0.01028366107493639, 0.007569293025881052, 0.015225829556584358, 0.0006035296246409416, 0.0014377726474776864, 0.018397418782114983, 0.00018186635861638933, 0.002113566268235445, 3.803668369073421e-05, 0.009962561540305614, 0.003998206928372383, 0.0012666863622143865, 0.0021862699650228024, 0.0032670684158802032, 0.0015871950890868902, 0.01913381926715374, 0.008779395371675491, 0.0017080748220905662, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5637838244438171, 2.204169322794769e-05, 0.0003808305482380092, 1.393800062032824e-07, 4.306913226059805e-08, 0.00012883225281257182, 5.202722968533635e-05, 4.098194494872587e-06, 0.00043821518192999065, 1.0102487067342736e-05, 2.0490055248956196e-05, 0.00021747536084149033, 2.5250110411434434e-05, 2.129377389792353e-05, 0.0022071911953389645, 5.892770423088223e-05, 0.002418374642729759, 0.003277554176747799, 0.42604929208755493, 0.0008841048693284392, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14635005593299866, 0.45109108090400696, 0.027205264195799828, 0.0030082776211202145, 0.0007913765148259699, 0.0800931379199028, 0.005927647929638624, 0.0006846337346360087, 0.0021268464624881744, 0.0027747468557208776, 0.00023907337163109332, 0.002550546545535326, 0.0054934462532401085, 0.01583220437169075, 0.0003450022195465863, 0.0005726688541471958, 0.002175167202949524, 0.039043255150318146, 0.16982686519622803, 0.04120763763785362, 0.0026610149070620537, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15883015096187592, 0.09413880854845047, 0.6926899552345276, 4.776390778715722e-05, 8.085349691100419e-06, 0.00935556460171938, 0.0008445517742075026, 2.443764515192015e-06, 0.00013773982936982065, 1.1189692941115936e-06, 4.677353445003973e-06, 0.0003472122480161488, 0.002631494775414467, 0.0004504133830778301, 0.006463215220719576, 0.0005723336944356561, 0.0012668337440118194, 0.006402322091162205, 0.0018092951504513621, 0.006555411033332348, 0.00037914939457550645, 0.017061391845345497, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05072391405701637, 0.0043561081402003765, 0.00013167233555577695, 0.9396469593048096, 0.0005500844563357532, 0.002771171974018216, 1.4556275345967151e-05, 5.0173721319879405e-06, 1.5498681023018435e-05, 7.020712899930004e-08, 8.694933967490215e-06, 3.654160536825657e-05, 3.607992539400584e-06, 2.5941648345906287e-05, 7.590860604977934e-06, 7.10092763256398e-07, 4.6298235247377306e-05, 7.143522088881582e-05, 0.00012089123629266396, 0.0005609994404949248, 1.3380947166297119e-05, 0.0007342157769016922, 0.0001547116116853431, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.044663090258836746, 0.0028326627798378468, 7.648707105545327e-05, 0.00015513667312916368, 0.7502228021621704, 0.19195663928985596, 9.640512871555984e-05, 0.00016210127796512097, 0.00012769461318384856, 1.1226569768041372e-05, 8.733285540074576e-06, 0.0002813320606946945, 5.2078037697356194e-05, 0.008386553265154362, 4.340289706306066e-06, 6.482724711531773e-05, 3.802955325227231e-05, 7.603636913700029e-05, 0.00012636324390769005, 9.227949340129271e-05, 4.0301659964825376e-07, 0.00011281677143415436, 2.522385329939425e-06, 0.0004493836604524404, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.013606224209070206, 0.006971816532313824, 3.1886105716694146e-05, 1.8455255030858098e-06, 0.002301084576174617, 0.9711790084838867, 0.0003632458101492375, 8.45977701828815e-05, 0.0001061156508512795, 4.5056845010549296e-07, 2.987579534874385e-07, 0.0008592646918259561, 8.844918920658529e-05, 0.00034812072408385575, 9.285179203288862e-07, 3.160578853567131e-05, 1.2802144738088828e-05, 5.8033951063407585e-05, 0.00010517514601815492, 6.438296259148046e-05, 1.8867377775677596e-06, 0.0009238758939318359, 4.68101461592596e-06, 8.99309179658303e-06, 0.0028452184051275253, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01961127668619156, 0.00438615120947361, 6.198724440764636e-05, 4.88576041846045e-08, 4.252470171195455e-05, 0.0036122461315244436, 0.9598402380943298, 0.005622219294309616, 0.0023445591796189547, 5.173985186956997e-07, 1.9621675164671615e-06, 0.00165482924785465, 0.0005915391957387328, 0.001169584458693862, 5.784227596450364e-06, 8.119036647258326e-05, 4.500302384258248e-05, 0.00018497802375350147, 5.086949386168271e-05, 0.00011132832878502086, 6.848460998298833e-06, 3.8539848901564255e-05, 4.853054633713327e-06, 1.276350758416811e-07, 3.953097802877892e-06, 0.0005269531393423676, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.008729088120162487, 0.0001781070022843778, 2.8806286422877747e-07, 4.138263136610476e-07, 6.864999886602163e-05, 0.0009331137989647686, 0.0001488832349423319, 0.984452486038208, 0.004786369390785694, 0.0001132894612965174, 7.255491141222592e-07, 7.424689101753756e-05, 1.4996417121437844e-05, 0.00019790598889812827, 2.995051318066544e-07, 4.872014415013837e-06, 5.296082690620096e-06, 4.536818778433371e-06, 0.00011268883827142417, 3.4170998333138414e-06, 8.272462764580268e-06, 1.23798881759285e-05, 3.665505587946427e-08, 9.0251188566981e-07, 1.6053050785558298e-05, 0.00011404338874854147, 1.8653627193998545e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04082155600190163, 0.003618426388129592, 4.821046968572773e-05, 1.9615674773376668e-07, 5.3802777983946726e-05, 0.0021802401170134544, 0.0033272523432970047, 0.001878200564533472, 0.9307873249053955, 0.00481536565348506, 1.443025394110009e-05, 0.0028287956956773996, 0.00010464123624842614, 0.0029309086967259645, 0.0009514776756986976, 6.522179319290444e-05, 0.00029541170806623995, 0.00012317505024839193, 0.0015512284589931369, 0.0005327722756192088, 0.0002208336372859776, 0.00044327162322588265, 1.9391492969589308e-05, 5.825700100103859e-07, 1.6632819097139873e-05, 0.0004773235123138875, 0.0011258182348683476, 0.0007676867535337806, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13800986111164093, 0.000338585494318977, 0.00015972793335095048, 3.422598027214008e-08, 1.2696453268290497e-05, 0.00024196873710025102, 3.82194593839813e-05, 3.751534677576274e-05, 0.004743305500596762, 0.8406771421432495, 0.00018875481327995658, 0.00015232685836963356, 1.250763853022363e-05, 0.0002236300497315824, 0.0001328253565588966, 6.470848165918142e-05, 0.00013950491847936064, 9.763600246515125e-05, 0.0004387965309433639, 4.486108446144499e-05, 6.358909013215452e-05, 0.00034256128128618, 5.2226907428121194e-05, 5.293519507176825e-07, 4.435913979250472e-06, 6.543510244227946e-05, 2.9145055577828316e-06, 3.8726907405362e-06, 0.013709785416722298, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0077987853437662125, 0.00022834629635326564, 4.638521033939469e-07, 6.367737910295546e-07, 1.5816805898793973e-05, 4.74391745228786e-05, 5.2391105782589875e-06, 7.306586212507682e-06, 1.252215042768512e-05, 3.330249569444277e-07, 0.9909055233001709, 0.00045220632455311716, 5.365398919821018e-06, 7.535894837928936e-05, 8.799969691608567e-06, 7.895076123531908e-06, 0.00023918210354167968, 2.256911329823197e-06, 7.641898992005736e-05, 5.514193617273122e-05, 4.378300673124613e-06, 2.181093987019267e-05, 4.435156242266203e-08, 1.2584397381942836e-06, 2.069822357952944e-06, 1.2951757526025176e-05, 3.952872305035271e-07, 1.1202362202311633e-06, 7.981805538292974e-06, 2.9978705242683645e-06, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04228433594107628, 0.01130534615367651, 7.640720696144854e-07, 2.2586132217838895e-06, 9.786387818166986e-05, 0.03826921060681343, 0.00023640895960852504, 0.001455514575354755, 0.0029497891664505005, 1.2863508800364798e-06, 2.8271693736314774e-05, 0.8946183323860168, 0.0005235012504272163, 0.0011799612548202276, 0.0009106284123845398, 0.0003604615631047636, 0.00020461656094994396, 2.910211514972616e-05, 0.0013425340875983238, 0.0003398252301849425, 0.00033254953450523317, 8.430417801719159e-05, 7.937666879342942e-08, 5.80987489229301e-06, 4.91085984322126e-06, 0.002387275919318199, 2.696716364880558e-05, 0.00020931514154653996, 0.0007688859477639198, 5.801381576020503e-06, 3.4147800761274993e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0061029596254229546, 0.016550516709685326, 4.6485445636790246e-05, 8.233629245069096e-08, 7.398499292321503e-06, 0.0005482283304445446, 0.0005931411869823933, 1.5738387446617708e-05, 3.503288098727353e-05, 3.3504278462714865e-07, 1.8657722478110372e-07, 0.00046066963113844395, 0.8416223526000977, 0.12311305105686188, 0.006359235383570194, 0.0026995246298611164, 0.00016676251834724098, 0.001213077805005014, 0.00013198796659708023, 8.468224405078217e-05, 4.615935722540598e-06, 6.994968862272799e-05, 4.916098987450823e-06, 3.4761444567266153e-07, 3.7119539797458856e-07, 6.107360968599096e-05, 6.728529115207493e-05, 2.2758390514354687e-06, 2.0601644791895524e-05, 7.198171942945919e-07, 2.78813168108627e-08, 1.6529560525668785e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04041679948568344, 0.001346431439742446, 0.0002881708787754178, 4.343534783401992e-06, 0.0006837420514784753, 0.0021492524538189173, 0.0012713877949863672, 0.00021145616483408958, 0.0001491332659497857, 2.3681526727159508e-05, 9.255093027604744e-05, 7.460857159458101e-05, 0.0030381479300558567, 0.9153464436531067, 0.002063496969640255, 0.002670764224603772, 0.0006971358670853078, 0.022915009409189224, 0.0016386171337217093, 0.0002943731960840523, 8.301850357383955e-06, 0.0001194091746583581, 3.610825660871342e-05, 1.2593762221513316e-05, 0.00038307331851683557, 0.0011952053755521774, 0.0001844509970396757, 7.125815318431705e-05, 0.00012850550410803407, 6.062198372092098e-05, 4.517644993029535e-05, 6.895749606883328e-07, 0.002379077021032572, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.03775661438703537, 0.0016132340533658862, 0.0003177856560796499, 8.17116472262569e-07, 4.6315386725837016e-07, 7.222060230560601e-05, 0.00012490959488786757, 4.99756515637273e-06, 0.00030667142709717155, 5.749376668973127e-06, 0.00024661491625010967, 0.003777766600251198, 0.0013358162250369787, 0.002113183494657278, 0.8997159600257874, 0.04417850822210312, 0.00033094940590672195, 0.0013849412789568305, 0.00039355995249934494, 0.002321068197488785, 0.000464222306618467, 0.00019271834753453732, 0.0005703889764845371, 5.927758593315957e-06, 2.4687206945372964e-08, 3.1083311569091165e-06, 7.165991064539412e-06, 2.9240266030683415e-07, 7.984187686815858e-05, 1.2546481229946949e-05, 0.00012651427823584527, 0.00023838109336793423, 0.0008907323936000466, 0.0014062426052987576, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08771780878305435, 0.00782146118581295, 5.6624849094077945e-05, 1.4731161179781793e-08, 1.0490798558748793e-05, 0.0011058745440095663, 0.00042911627679131925, 5.52300589333754e-05, 3.919872688129544e-05, 2.7502203465701314e-06, 2.5461979475949192e-06, 0.002755212364718318, 0.005722680129110813, 0.0011191823286935687, 0.0015876379329711199, 0.8757358193397522, 0.001080039655789733, 0.0035321444738656282, 0.0015019761631265283, 0.0006949526141397655, 4.334090044721961e-05, 0.00023435342882294208, 6.934279099368723e-06, 2.757802519681718e-08, 9.912479299600818e-07, 0.00019156596681568772, 2.9515393180190586e-05, 5.468214112624992e-06, 2.480577677488327e-06, 1.4659925682281028e-06, 7.256068101924029e-07, 8.567866461817175e-05, 0.004263878334313631, 0.0007048743427731097, 0.0034579597413539886, 0.0, 0.0, 0.0, 0.0, 0.0], [0.018741386011242867, 0.0001317748101428151, 0.00011519038525875658, 8.336405130648927e-07, 3.873432774526009e-07, 4.0343722503166646e-05, 8.927313319873065e-05, 6.575927727681119e-06, 6.646558176726103e-05, 2.0145262169535272e-07, 0.00012498456635512412, 0.00022737613471690565, 0.00035967829171568155, 4.087335037183948e-05, 8.65399488247931e-05, 0.00017062197730410844, 0.9720673561096191, 0.004511047620326281, 0.0017741499468684196, 0.00033421028638258576, 0.0006673677125945687, 8.272402737929951e-06, 8.590817014919594e-05, 4.445200545433181e-07, 1.5395107411109166e-08, 3.916216883226298e-06, 1.8561571778263897e-05, 1.7440333976992406e-06, 2.306471833435353e-05, 6.561277814398636e-07, 5.536517346627079e-05, 1.0027780263044406e-05, 0.00011005577107425779, 2.0988292817492038e-05, 6.0086065786890686e-05, 4.435596929397434e-05, 0.0, 0.0, 0.0, 0.0], [0.02161449007689953, 6.699986261082813e-05, 0.00023383108782581985, 1.9461008093912824e-07, 1.05680453543755e-06, 2.569968273746781e-05, 0.00021046775509603322, 1.7352506347378949e-06, 6.270146968745394e-06, 8.374834692403965e-08, 2.354792513870052e-06, 1.806468026188668e-05, 0.00023376732133328915, 0.0018953320104628801, 0.00020409566059242934, 0.0004746932827401906, 0.0029953878838568926, 0.962553858757019, 0.0069142491556704044, 0.0001976252970052883, 0.00011761276255128905, 1.7810549252317287e-05, 8.193789835786447e-05, 8.319970135062249e-08, 6.341087299688297e-08, 1.5034723901408142e-06, 1.5951758541632444e-05, 1.285200426082156e-07, 7.935356052257703e-07, 1.2686608386047737e-07, 5.14019859565451e-07, 3.882940191601847e-08, 5.885375867364928e-05, 0.0011026912834495306, 9.71672561718151e-05, 8.231383981183171e-05, 0.0007721207803115249, 0.0, 0.0, 0.0], [0.036683376878499985, 5.292138666845858e-06, 1.6796333284219145e-06, 3.225949229967995e-10, 1.7679223418554102e-08, 9.711849997984245e-06, 6.97010455041891e-06, 3.922056293959031e-06, 1.5272446034941822e-05, 6.423179002013057e-07, 3.00263849339899e-07, 2.1527906937990338e-05, 8.620364724265528e-07, 2.37110707530519e-05, 1.133400132857787e-06, 4.3700861169782e-06, 2.1794843632960692e-05, 0.0007244828739203513, 0.956550121307373, 7.606161670992151e-05, 0.005361158866435289, 3.409740747883916e-05, 1.4788669204790494e-06, 1.967607188291254e-09, 6.515270101203896e-09, 7.016132030912559e-07, 1.9082037283624231e-07, 3.6050772678208887e-07, 3.973362709075445e-06, 1.5304426597140264e-06, 1.0371004321996224e-07, 7.635152314833249e-07, 1.8985033989338262e-07, 6.164461410662625e-06, 2.613814103824552e-06, 1.352905542262306e-06, 1.0274186024616938e-05, 0.0004238302353769541, 0.0, 0.0], [0.03937007114291191, 0.00035857962211593986, 7.419281610054895e-05, 4.950996662955731e-05, 4.6530603867722675e-06, 4.697596523328684e-05, 0.00022992271988186985, 1.2504364121923572e-06, 3.72327740478795e-05, 8.911823101698246e-08, 7.931985601317137e-05, 0.000135091133415699, 1.8374767023487948e-05, 0.00010297342669218779, 8.208450162783265e-05, 2.753637636487838e-05, 0.0005808338755741715, 0.0011750266421586275, 0.0009059829753823578, 0.9545682072639465, 7.417569577228278e-05, 0.0005086683668196201, 0.00019448318926151842, 8.633160177851096e-05, 8.95263099209842e-07, 1.4289948921941686e-05, 3.445189940975979e-05, 2.1258644267163618e-07, 1.6950663848547265e-05, 4.816170076082926e-07, 3.18538659485057e-05, 1.3589556147053372e-05, 2.5087487301789224e-05, 5.9819085436174646e-05, 0.0001169012684840709, 1.1341999197611585e-05, 0.0002713195572141558, 0.0002638357982505113, 0.00042732004658319056, 0.0], [0.025401176884770393, 7.441661864504567e-07, 1.0830090104718693e-05, 2.2497226304096785e-09, 9.221913788159952e-10, 8.208573490264826e-06, 2.5448041469644522e-06, 1.0004280426301193e-07, 3.160869891871698e-05, 2.4278301680169534e-07, 5.289186333357065e-07, 1.4262926924857311e-05, 1.5345807469202555e-06, 1.87778505278402e-06, 0.0002028680028161034, 2.1412402020359877e-06, 0.00010114445467479527, 0.0009309852030128241, 0.1016610637307167, 5.551521462621167e-05, 0.8627558350563049, 2.250279521831544e-06, 3.066586941713467e-05, 5.196949981467469e-09, 3.1991624627192294e-11, 8.736405021636529e-08, 1.2241659419487405e-07, 1.2374152813521277e-09, 1.2507486644608434e-05, 4.635763275473437e-07, 1.6297862259762042e-07, 2.719466181133612e-07, 1.1941818911509472e-07, 1.4462575848028791e-07, 5.527890243683942e-05, 3.5639450857161137e-07, 4.933351738145575e-05, 0.0001694788079475984, 0.008456701412796974, 3.871364606311545e-05]]]}
     )
@@ -1029,37 +1029,37 @@ <h2>Available Models<a class="headerlink" href="#Available-Models" title="Permal
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "2b653bed330e47bfaf1eb97604ffb148"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "16f9e028019c47c19107395141087d9c"}</script></div>
 </div>
 <div class="nboutput docutils container">
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "2f95bda34e124bdb8ad42b97968c2560"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "8c41c6eefd75443fa69c9327b4c1ec8e"}</script></div>
 </div>
 <div class="nboutput docutils container">
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "1bfe4dc8f9d64ed3a70a0e6e41b61d34"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "6a66981f155d4bf280913b398c75f705"}</script></div>
 </div>
 <div class="nboutput docutils container">
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "363a53f62bad40f38d9b57415fcdd27a"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "59a6916e36b44608987b585683320e55"}</script></div>
 </div>
 <div class="nboutput docutils container">
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "baa802cac1bf4a77bd2d6f2da4e0c206"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "f462bf0c8aef4d9e9fd54629b8fe4434"}</script></div>
 </div>
 <div class="nboutput docutils container">
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "85239cb8d6ba4d288b001290e919d816"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "29b6d4431334417e9690be62ac184ff3"}</script></div>
 </div>
 <div class="nboutput nblast docutils container">
 <div class="prompt empty docutils container">
@@ -1111,9 +1111,9 @@ <h2>Available Models<a class="headerlink" href="#Available-Models" title="Permal
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="e151a52d-4559-414d-bbd0-4a08a83b57c6" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("e151a52d-4559-414d-bbd0-4a08a83b57c6")) {                    Plotly.newPlot(                        "e151a52d-4559-414d-bbd0-4a08a83b57c6",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.010150859132409096,0.00021067557099740952,0.011675543151795864,5.645640158036258e-06,0.0009093668777495623,1.0044651389762294e-05,0.008776976726949215,0.0014797636540606618,0.008503900840878487,0.009458577260375023,0.009176097810268402,0.015341502614319324],[0.0030316568445414305,0.018613651394844055,0.003521384671330452,0.0002105814783135429,0.012771016918122768,0.002262213034555316,0.005199276376515627,0.015398663468658924,0.001596646150574088,7.471267599612474e-05,0.004113694187253714,0.014634193852543831],[0.00835011899471283,0.005591983906924725,0.01345061045140028,0.0016680879052728415,0.019946973770856857,0.004370617680251598,0.009695201180875301,0.001158412080258131,0.01791170984506607,0.01292894221842289,0.021461093798279762,5.112248818717191e-11],[0.005671565420925617,0.20814286172389984,0.8488407135009766,0.01734151318669319,0.017860308289527893,0.014111736789345741,0.020909568294882774,0.1804729551076889,0.01673540472984314,0.0179011020809412,0.9288043975830078,0.4491482973098755],[0.26883465051651,0.23360644280910492,0.08689460158348083,0.011528643779456615,0.07701855152845383,0.024650055915117264,0.6229627132415771,0.025258244946599007,0.06073903664946556,0.6466246247291565,0.017515525221824646,0.054975464940071106],[0.021400248631834984,0.07713880389928818,0.05230269953608513,0.013922988437116146,0.032270368188619614,0.18037253618240356,0.07306526601314545,0.0951840952038765,0.007617602590471506,0.4484187066555023,0.17469297349452972,0.03143536299467087]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Induction Score by Head in Distil GPT-2"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="e0327d47-bf11-4c31-89f2-acefb328a7c7" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("e0327d47-bf11-4c31-89f2-acefb328a7c7")) {                    Plotly.newPlot(                        "e0327d47-bf11-4c31-89f2-acefb328a7c7",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.010554764419794083,0.00017109278996940702,0.01157795637845993,2.9588852612505434e-06,0.0007345465710386634,1.1151450053148437e-05,0.00925703626126051,0.001245542080141604,0.00874336902052164,0.009286121465265751,0.009252987802028656,0.01566421426832676],[0.0027158448938280344,0.016322769224643707,0.0025369308423250914,0.0002154479589080438,0.015138639137148857,0.0016251271590590477,0.004624444060027599,0.015645790845155716,0.0013522073859348893,4.790473394677974e-05,0.003898467170074582,0.014824452809989452],[0.009688482619822025,0.007363068405538797,0.011820577085018158,0.0032990179024636745,0.01895608939230442,0.004869616124778986,0.009377731941640377,0.0014279317110776901,0.017006751149892807,0.012095690704882145,0.021088911220431328,8.396961348466903e-09],[0.00907871313393116,0.23671726882457733,0.8545374274253845,0.016443107277154922,0.01815449446439743,0.013023864477872849,0.01957394927740097,0.19624365866184235,0.01637774519622326,0.017492501065135002,0.930077075958252,0.4885671138763428],[0.2790757715702057,0.22406823933124542,0.0913514792919159,0.012907207943499088,0.08906538784503937,0.025342751294374466,0.6347011923789978,0.025462210178375244,0.06470905244350433,0.648553192615509,0.01796729676425457,0.06295257061719894],[0.02277965098619461,0.07785113155841827,0.05185911804437637,0.013325891457498074,0.035574771463871,0.17566177248954773,0.07605704665184021,0.09583288431167603,0.008021574467420578,0.45598646998405457,0.20273584127426147,0.03064807876944542]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Induction Score by Head in Distil GPT-2"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('e151a52d-4559-414d-bbd0-4a08a83b57c6');
+var gd = document.getElementById('e0327d47-bf11-4c31-89f2-acefb328a7c7');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -1566,19 +1566,12 @@ <h2>Dealing with tokens<a class="headerlink" href="#Dealing-with-tokens" title="
 </pre></div>
 </div>
 </div>
-<div class="nboutput docutils container">
-<div class="prompt empty docutils container">
-</div>
-<div class="output_area docutils container">
-<div class="highlight"><pre>
-Probability tensor shape [batch, position, d_vocab] == torch.Size([1, 8, 50257])
-</pre></div></div>
-</div>
 <div class="nboutput nblast docutils container">
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
 <div class="highlight"><pre>
+Probability tensor shape [batch, position, d_vocab] == torch.Size([1, 8, 50257])
 | The| probability: 11.98%
 </pre></div></div>
 </div>
@@ -1740,20 +1733,12 @@ <h3>Gotcha: <code class="docutils literal notranslate"><span class="pre">prepend
 Logit difference with BOS: 6.754
 </pre></div></div>
 </div>
-<div class="nboutput docutils container">
-<div class="prompt empty docutils container">
-</div>
-<div class="output_area docutils container">
-<div class="highlight"><pre>
-Logit difference without BOS: 2.782
-</pre></div></div>
-</div>
 <div class="nboutput nblast docutils container">
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
 <div class="highlight"><pre>
-
+Logit difference without BOS: 2.782
 </pre></div></div>
 </div>
 <p>Though, note that this also illustrates another gotcha - when <code class="docutils literal notranslate"><span class="pre">Claire</span></code> is at the start of a sentence (no preceding space), it’s actually <em>two</em> tokens, not one, which probably confuses the relevant circuit. (Note - in this test we put <code class="docutils literal notranslate"><span class="pre">prepend_bos=False</span></code>, because we want to analyse the tokenization of a specific string, not to give an input to the model!)</p>
@@ -1960,9 +1945,9 @@ <h3>Medium Example: Eigenvalue Copying Scores<a class="headerlink" href="#Medium
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="458e8707-ccf3-4567-a8b0-f931c572b246" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("458e8707-ccf3-4567-a8b0-f931c572b246")) {                    Plotly.newPlot(                        "458e8707-ccf3-4567-a8b0-f931c572b246",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.7775008678436279,0.35272663831710815,0.25961872935295105,0.6670258045196533,0.838425874710083,0.5584433078765869,0.844474732875824,0.41379114985466003,0.244889497756958,0.02815753035247326,0.3584096431732178,0.16288283467292786],[-0.4541913866996765,-0.6529324650764465,-0.5484569668769836,-0.7990368604660034,-0.7736426591873169,-0.8522582650184631,0.977432370185852,0.6626251935958862,-0.7303223609924316,-0.7007019519805908,-0.6946623921394348,-0.9996723532676697],[-0.7837162613868713,0.8967758417129517,0.4750956892967224,-0.6671973466873169,0.7881463170051575,-0.8547748923301697,-0.9054183959960938,-0.5749384760856628,-0.321751207113266,-0.0285941194742918,-0.9247617125511169,-0.9699268937110901],[0.5864036083221436,-0.76143479347229,0.5971695184707642,0.7854391932487488,-0.8788884878158569,0.3908745348453522,0.0447387658059597,0.1102800965309143,-0.8169987201690674,0.2212953418493271,-0.9939578771591187,0.5774401426315308],[0.5254791378974915,0.30490121245384216,-0.10729170590639114,0.9433152675628662,-0.9314427971839905,0.5273631811141968,-0.4264712929725647,-0.9984429478645325,0.5296757817268372,0.8604294657707214,-0.8895052075386047,0.9556970000267029],[0.6629188656806946,0.4295697808265686,0.9736858010292053,0.655548095703125,0.12201863527297974,0.7442769408226013,0.5037954449653625,0.95253586769104,-0.6507166624069214,-0.9316278100013733,0.9791510701179504,-0.9972586035728455],[0.9613031148910522,0.7501779794692993,-0.3806658983230591,0.6429785490036011,0.9557769298553467,-0.9428837895393372,-0.9948079586029053,0.7852989435195923,0.9657301306724548,0.7073014974594116,0.36872273683547974,0.8128010034561157],[0.9659482836723328,0.9730120301246643,0.31900620460510254,-0.30290529131889343,0.9790952801704407,0.9357922673225403,-0.5550314784049988,-0.0054661668837070465,0.986777663230896,0.8249568343162537,0.5664296746253967,0.1000528484582901],[-0.9464486837387085,-0.25471991300582886,0.6522327065467834,0.14152583479881287,0.9884141683578491,0.9860584735870361,0.6949271559715271,0.9901811480522156,0.9791203141212463,-0.23595543205738068,-0.982071042060852,0.6506688594818115],[0.9895945191383362,-0.291781485080719,0.9714024662971497,0.9951602220535278,0.18783727288246155,-0.9460937976837158,0.4780191481113434,-0.2489192634820938,0.9437099099159241,0.11866225302219391,0.9941242933273315,-0.38088199496269226],[0.9564487338066101,0.5542722344398499,0.42118069529533386,0.6628788113594055,0.8659593462944031,0.9937117695808411,0.9069075584411621,0.3981107473373413,-0.4134218096733093,0.9971914887428284,0.3459664583206177,0.9938657283782959],[0.5891268253326416,0.9313738942146301,0.9268401861190796,0.9993563890457153,0.6227542161941528,0.8463947772979736,0.6584343910217285,0.8423123955726624,0.2978499233722687,0.8728678822517395,0.9963143467903137,0.9867526292800903]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0,"cmin":-1.0,"cmax":1.0},"title":{"text":"OV Copying Score for each head in GPT-2 Small"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="d3f848b1-d2b5-480a-aea1-fbcc08758557" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("d3f848b1-d2b5-480a-aea1-fbcc08758557")) {                    Plotly.newPlot(                        "d3f848b1-d2b5-480a-aea1-fbcc08758557",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.7775008678436279,0.35272663831710815,0.25961872935295105,0.6670258045196533,0.838425874710083,0.5584433078765869,0.844474732875824,0.41379114985466003,0.244889497756958,0.02815753035247326,0.3584096431732178,0.16288283467292786],[-0.4541913866996765,-0.6529324650764465,-0.5484569668769836,-0.7990368604660034,-0.7736426591873169,-0.8522582650184631,0.977432370185852,0.6626251935958862,-0.7303223609924316,-0.7007019519805908,-0.6946623921394348,-0.9996723532676697],[-0.7837162613868713,0.8967758417129517,0.4750956892967224,-0.6671973466873169,0.7881463170051575,-0.8547748923301697,-0.9054183959960938,-0.5749384760856628,-0.321751207113266,-0.0285941194742918,-0.9247617125511169,-0.9699268937110901],[0.5864036083221436,-0.76143479347229,0.5971695184707642,0.7854391932487488,-0.8788884878158569,0.3908745348453522,0.0447387658059597,0.1102800965309143,-0.8169987201690674,0.2212953418493271,-0.9939578771591187,0.5774401426315308],[0.5254791378974915,0.30490121245384216,-0.10729170590639114,0.9433152675628662,-0.9314427971839905,0.5273631811141968,-0.4264712929725647,-0.9984429478645325,0.5296757817268372,0.8604294657707214,-0.8895052075386047,0.9556970000267029],[0.6629188656806946,0.4295697808265686,0.9736858010292053,0.655548095703125,0.12201863527297974,0.7442769408226013,0.5037954449653625,0.95253586769104,-0.6507166624069214,-0.9316278100013733,0.9791510701179504,-0.9972586035728455],[0.9613031148910522,0.7501779794692993,-0.3806658983230591,0.6429785490036011,0.9557769298553467,-0.9428837895393372,-0.9948079586029053,0.7852989435195923,0.9657301306724548,0.7073014974594116,0.36872273683547974,0.8128010034561157],[0.9659482836723328,0.9730120301246643,0.31900620460510254,-0.30290529131889343,0.9790952801704407,0.9357922673225403,-0.5550314784049988,-0.0054661668837070465,0.986777663230896,0.8249568343162537,0.5664296746253967,0.1000528484582901],[-0.9464486837387085,-0.25471991300582886,0.6522327065467834,0.14152583479881287,0.9884141683578491,0.9860584735870361,0.6949271559715271,0.9901811480522156,0.9791203141212463,-0.23595543205738068,-0.982071042060852,0.6506688594818115],[0.9895945191383362,-0.291781485080719,0.9714024662971497,0.9951602220535278,0.18783727288246155,-0.9460937976837158,0.4780191481113434,-0.2489192634820938,0.9437099099159241,0.11866225302219391,0.9941242933273315,-0.38088199496269226],[0.9564487338066101,0.5542722344398499,0.42118069529533386,0.6628788113594055,0.8659593462944031,0.9937117695808411,0.9069075584411621,0.3981107473373413,-0.4134218096733093,0.9971914887428284,0.3459664583206177,0.9938657283782959],[0.5891268253326416,0.9313738942146301,0.9268401861190796,0.9993563890457153,0.6227542161941528,0.8463947772979736,0.6584343910217285,0.8423123955726624,0.2978499233722687,0.8728678822517395,0.9963143467903137,0.9867526292800903]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0,"cmin":-1.0,"cmax":1.0},"title":{"text":"OV Copying Score for each head in GPT-2 Small"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('458e8707-ccf3-4567-a8b0-f931c572b246');
+var gd = document.getElementById('d3f848b1-d2b5-480a-aea1-fbcc08758557');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2005,9 +1990,9 @@ <h3>Medium Example: Eigenvalue Copying Scores<a class="headerlink" href="#Medium
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="79eef653-2ecc-4731-aedd-e4b748085838" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("79eef653-2ecc-4731-aedd-e4b748085838")) {                    Plotly.newPlot(                        "79eef653-2ecc-4731-aedd-e4b748085838",                        [{"hovertemplate":"Real=%{x}\u003cbr\u003eImaginary=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"","marker":{"color":"#636efa","symbol":"circle"},"mode":"markers","name":"","orientation":"v","showlegend":false,"x":[-2.139726161956787,1.4152659177780151,3.4444546699523926,4.0276689529418945,8.882657051086426,4.866776943206787,4.866776943206787,4.843708038330078,4.843708038330078,8.477534294128418,8.21679973602295,8.21679973602295,5.078621864318848,7.855459213256836,7.855459213256836,5.365771770477295,5.365771770477295,5.563427448272705,5.563427448272705,5.421727657318115,7.769131660461426,7.769131660461426,7.042290210723877,7.042290210723877,5.675149917602539,5.675149917602539,7.6785993576049805,7.6785993576049805,6.573331356048584,6.573331356048584,7.67292594909668,7.172202110290527,7.172202110290527,7.423615455627441,7.423615455627441,7.4708147048950195,6.08908748626709,6.08908748626709,6.306832313537598,6.306832313537598,6.511747360229492,6.511747360229492,5.955250263214111,5.955250263214111,5.858802795410156,5.858802795410156,7.147878170013428,7.147878170013428,7.185699939727783,7.185699939727783,6.6706037521362305,6.6706037521362305,6.7359819412231445,6.7359819412231445,6.149754524230957,6.149754524230957,6.288780212402344,6.288780212402344,6.344789028167725,6.625576972961426,6.625576972961426,6.899187088012695,6.899187088012695,6.856410026550293],"xaxis":"x","y":[0.0,0.0,0.0,0.0,0.0,0.41851967573165894,-0.41851967573165894,0.09079498052597046,-0.09079498052597046,0.0,0.40868431329727173,-0.40868431329727173,0.0,0.7007191777229309,-0.7007191777229309,0.46421778202056885,-0.46421778202056885,0.5558270215988159,-0.5558270215988159,0.0,0.47056713700294495,-0.47056713700294495,1.029872179031372,-1.029872179031372,0.48252731561660767,-0.48252731561660767,0.33564886450767517,-0.33564886450767517,0.9988701343536377,-0.9988701343536377,0.0,0.7531797885894775,-0.7531797885894775,0.425758957862854,-0.425758957862854,0.0,0.6436269283294678,-0.6436269283294678,0.7701666951179504,-0.7701666951179504,0.7558029294013977,-0.7558029294013977,0.2591177523136139,-0.2591177523136139,0.013043581508100033,-0.013043581508100033,0.40166139602661133,-0.40166139602661133,0.28192487359046936,-0.28192487359046936,0.6146230697631836,-0.6146230697631836,0.5391324758529663,-0.5391324758529663,0.2823374569416046,-0.2823374569416046,0.35283783078193665,-0.35283783078193665,0.0,0.24868342280387878,-0.24868342280387878,0.15545885264873505,-0.15545885264873505,0.0],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"Real"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"Imaginary"}},"legend":{"tracegroupgap":0},"title":{"text":"Eigenvalues of Head L11H11 of GPT-2 Small"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="e046ce15-f732-4def-bd7e-56de079b2f60" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("e046ce15-f732-4def-bd7e-56de079b2f60")) {                    Plotly.newPlot(                        "e046ce15-f732-4def-bd7e-56de079b2f60",                        [{"hovertemplate":"Real=%{x}\u003cbr\u003eImaginary=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"","marker":{"color":"#636efa","symbol":"circle"},"mode":"markers","name":"","orientation":"v","showlegend":false,"x":[-2.139726161956787,1.4152659177780151,3.4444546699523926,4.0276689529418945,8.882657051086426,4.866776943206787,4.866776943206787,4.843708038330078,4.843708038330078,8.477534294128418,8.21679973602295,8.21679973602295,5.078621864318848,7.855459213256836,7.855459213256836,5.365771770477295,5.365771770477295,5.563427448272705,5.563427448272705,5.421727657318115,7.769131660461426,7.769131660461426,7.042290210723877,7.042290210723877,5.675149917602539,5.675149917602539,7.6785993576049805,7.6785993576049805,6.573331356048584,6.573331356048584,7.67292594909668,7.172202110290527,7.172202110290527,7.423615455627441,7.423615455627441,7.4708147048950195,6.08908748626709,6.08908748626709,6.306832313537598,6.306832313537598,6.511747360229492,6.511747360229492,5.955250263214111,5.955250263214111,5.858802795410156,5.858802795410156,7.147878170013428,7.147878170013428,7.185699939727783,7.185699939727783,6.6706037521362305,6.6706037521362305,6.7359819412231445,6.7359819412231445,6.149754524230957,6.149754524230957,6.288780212402344,6.288780212402344,6.344789028167725,6.625576972961426,6.625576972961426,6.899187088012695,6.899187088012695,6.856410026550293],"xaxis":"x","y":[0.0,0.0,0.0,0.0,0.0,0.41851967573165894,-0.41851967573165894,0.09079498052597046,-0.09079498052597046,0.0,0.40868431329727173,-0.40868431329727173,0.0,0.7007191777229309,-0.7007191777229309,0.46421778202056885,-0.46421778202056885,0.5558270215988159,-0.5558270215988159,0.0,0.47056713700294495,-0.47056713700294495,1.029872179031372,-1.029872179031372,0.48252731561660767,-0.48252731561660767,0.33564886450767517,-0.33564886450767517,0.9988701343536377,-0.9988701343536377,0.0,0.7531797885894775,-0.7531797885894775,0.425758957862854,-0.425758957862854,0.0,0.6436269283294678,-0.6436269283294678,0.7701666951179504,-0.7701666951179504,0.7558029294013977,-0.7558029294013977,0.2591177523136139,-0.2591177523136139,0.013043581508100033,-0.013043581508100033,0.40166139602661133,-0.40166139602661133,0.28192487359046936,-0.28192487359046936,0.6146230697631836,-0.6146230697631836,0.5391324758529663,-0.5391324758529663,0.2823374569416046,-0.2823374569416046,0.35283783078193665,-0.35283783078193665,0.0,0.24868342280387878,-0.24868342280387878,0.15545885264873505,-0.15545885264873505,0.0],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"Real"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"Imaginary"}},"legend":{"tracegroupgap":0},"title":{"text":"Eigenvalues of Head L11H11 of GPT-2 Small"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('79eef653-2ecc-4731-aedd-e4b748085838');
+var gd = document.getElementById('e046ce15-f732-4def-bd7e-56de079b2f60');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2087,9 +2072,9 @@ <h3>Medium Example: Eigenvalue Copying Scores<a class="headerlink" href="#Medium
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="b62ac579-268a-4cb8-a4db-de9c144e5855" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("b62ac579-268a-4cb8-a4db-de9c144e5855")) {                    Plotly.newPlot(                        "b62ac579-268a-4cb8-a4db-de9c144e5855",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.8356367349624634,0.5853534936904907,0.5105841159820557,0.7843377590179443,0.8644161224365234,0.7026589512825012,0.8969924449920654,0.5868823528289795,0.42486509680747986,-0.16337518393993378,0.46268585324287415,0.276053786277771],[-0.052920110523700714,-0.31773144006729126,-0.4810579717159271,-0.7838066220283508,-0.6360211968421936,-0.77586829662323,0.9681803584098816,0.8119116425514221,-0.7510464787483215,-0.6878445744514465,-0.6429887413978577,-0.9985856413841248],[-0.6598325967788696,0.9152501821517944,0.5461500883102417,-0.48743969202041626,0.7720563411712646,-0.7541061639785767,-0.8472450971603394,-0.6948987245559692,-0.1557510495185852,0.2444225549697876,-0.9106622338294983,-0.9439151287078857],[0.648689866065979,-0.5592910647392273,0.5935593247413635,0.7843040823936462,-0.8150346279144287,0.613004744052887,0.16785882413387299,0.35195910930633545,-0.6837262511253357,0.2223764955997467,-0.9929219484329224,0.6535818576812744],[0.5740953087806702,0.3640134036540985,0.09609051048755646,0.9359624981880188,-0.9228776097297668,0.6191076636314392,-0.33572638034820557,-0.998464822769165,0.6448632478713989,0.8468661308288574,-0.7557656764984131,0.9527971744537354],[0.7326544523239136,0.5324169397354126,0.9732670187950134,0.7239246964454651,0.2553895115852356,0.8158416152000427,0.665579080581665,0.9287100434303284,-0.5660436153411865,-0.8908745050430298,0.9834234118461609,-0.9981179237365723],[0.9698692560195923,0.7439671754837036,-0.35639333724975586,0.6022987365722656,0.9708116054534912,-0.9278275966644287,-0.9962316155433655,0.834520697593689,0.9714328050613403,0.8158544898033142,0.5902575850486755,0.8199343681335449],[0.9820227026939392,0.9859329462051392,0.5152460932731628,-0.5610517263412476,0.9663666486740112,0.9495157599449158,-0.5204814076423645,0.3104752004146576,0.9859083890914917,0.7797460556030273,0.6738530397415161,0.39197394251823425],[-0.906204104423523,0.11750960350036621,0.8077874779701233,0.416930615901947,0.9829014539718628,0.9902300834655762,0.7847104668617249,0.9945629835128784,0.9868024587631226,-0.26804426312446594,-0.9908867478370667,0.745792806148529],[0.9906190633773804,-0.18231146037578583,0.9757838249206543,0.9986750483512878,0.2544330954551697,-0.954406201839447,0.586924135684967,-0.23537978529930115,0.9550502896308899,0.2551196813583374,0.9929869771003723,0.0905260294675827],[0.9707273244857788,0.6956090927124023,0.6280022263526917,0.7902868390083313,0.9343841671943665,0.9895793795585632,0.9436282515525818,-0.10834990441799164,-0.3431110680103302,0.9986709952354431,0.508673906326294,0.9949509501457214],[0.8283132910728455,0.9432437419891357,0.9491764903068542,0.9995353817939758,0.5712320804595947,0.8055236339569092,0.6781865954399109,0.8272573351860046,0.8314797282218933,0.8778655529022217,0.9944959282875061,0.997386634349823]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0,"cmin":-1.0,"cmax":1.0},"title":{"text":"OV Copying Score for each head in GPT-2 Small"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="bac66523-6094-49ba-be5d-8fd8a0f66934" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("bac66523-6094-49ba-be5d-8fd8a0f66934")) {                    Plotly.newPlot(                        "bac66523-6094-49ba-be5d-8fd8a0f66934",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.8356367349624634,0.5853534936904907,0.5105841159820557,0.7843377590179443,0.8644161224365234,0.7026589512825012,0.8969924449920654,0.5868823528289795,0.42486509680747986,-0.16337518393993378,0.46268585324287415,0.276053786277771],[-0.052920110523700714,-0.31773144006729126,-0.4810579717159271,-0.7838066220283508,-0.6360211968421936,-0.77586829662323,0.9681803584098816,0.8119116425514221,-0.7510464787483215,-0.6878445744514465,-0.6429887413978577,-0.9985856413841248],[-0.6598325967788696,0.9152501821517944,0.5461500883102417,-0.48743969202041626,0.7720563411712646,-0.7541061639785767,-0.8472450971603394,-0.6948987245559692,-0.1557510495185852,0.2444225549697876,-0.9106622338294983,-0.9439151287078857],[0.648689866065979,-0.5592910647392273,0.5935593247413635,0.7843040823936462,-0.8150346279144287,0.613004744052887,0.16785882413387299,0.35195910930633545,-0.6837262511253357,0.2223764955997467,-0.9929219484329224,0.6535818576812744],[0.5740953087806702,0.3640134036540985,0.09609051048755646,0.9359624981880188,-0.9228776097297668,0.6191076636314392,-0.33572638034820557,-0.998464822769165,0.6448632478713989,0.8468661308288574,-0.7557656764984131,0.9527971744537354],[0.7326544523239136,0.5324169397354126,0.9732670187950134,0.7239246964454651,0.2553895115852356,0.8158416152000427,0.665579080581665,0.9287100434303284,-0.5660436153411865,-0.8908745050430298,0.9834234118461609,-0.9981179237365723],[0.9698692560195923,0.7439671754837036,-0.35639333724975586,0.6022987365722656,0.9708116054534912,-0.9278275966644287,-0.9962316155433655,0.834520697593689,0.9714328050613403,0.8158544898033142,0.5902575850486755,0.8199343681335449],[0.9820227026939392,0.9859329462051392,0.5152460932731628,-0.5610517263412476,0.9663666486740112,0.9495157599449158,-0.5204814076423645,0.3104752004146576,0.9859083890914917,0.7797460556030273,0.6738530397415161,0.39197394251823425],[-0.906204104423523,0.11750960350036621,0.8077874779701233,0.416930615901947,0.9829014539718628,0.9902300834655762,0.7847104668617249,0.9945629835128784,0.9868024587631226,-0.26804426312446594,-0.9908867478370667,0.745792806148529],[0.9906190633773804,-0.18231146037578583,0.9757838249206543,0.9986750483512878,0.2544330954551697,-0.954406201839447,0.586924135684967,-0.23537978529930115,0.9550502896308899,0.2551196813583374,0.9929869771003723,0.0905260294675827],[0.9707273244857788,0.6956090927124023,0.6280022263526917,0.7902868390083313,0.9343841671943665,0.9895793795585632,0.9436282515525818,-0.10834990441799164,-0.3431110680103302,0.9986709952354431,0.508673906326294,0.9949509501457214],[0.8283132910728455,0.9432437419891357,0.9491764903068542,0.9995353817939758,0.5712320804595947,0.8055236339569092,0.6781865954399109,0.8272573351860046,0.8314797282218933,0.8778655529022217,0.9944959282875061,0.997386634349823]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0,"cmin":-1.0,"cmax":1.0},"title":{"text":"OV Copying Score for each head in GPT-2 Small"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('b62ac579-268a-4cb8-a4db-de9c144e5855');
+var gd = document.getElementById('bac66523-6094-49ba-be5d-8fd8a0f66934');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2132,9 +2117,9 @@ <h3>Medium Example: Eigenvalue Copying Scores<a class="headerlink" href="#Medium
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="3c8c77e9-6054-4886-85c5-8868e8e7c01c" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("3c8c77e9-6054-4886-85c5-8868e8e7c01c")) {                    Plotly.newPlot(                        "3c8c77e9-6054-4886-85c5-8868e8e7c01c",                        [{"hovertemplate":"\u003cb\u003e%{hovertext}\u003c\u002fb\u003e\u003cbr\u003e\u003cbr\u003eFull OV Copying Score=%{x}\u003cbr\u003eOV Copying Score=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","hovertext":["L0H0","L0H1","L0H2","L0H3","L0H4","L0H5","L0H6","L0H7","L0H8","L0H9","L0H10","L0H11","L1H0","L1H1","L1H2","L1H3","L1H4","L1H5","L1H6","L1H7","L1H8","L1H9","L1H10","L1H11","L2H0","L2H1","L2H2","L2H3","L2H4","L2H5","L2H6","L2H7","L2H8","L2H9","L2H10","L2H11","L3H0","L3H1","L3H2","L3H3","L3H4","L3H5","L3H6","L3H7","L3H8","L3H9","L3H10","L3H11","L4H0","L4H1","L4H2","L4H3","L4H4","L4H5","L4H6","L4H7","L4H8","L4H9","L4H10","L4H11","L5H0","L5H1","L5H2","L5H3","L5H4","L5H5","L5H6","L5H7","L5H8","L5H9","L5H10","L5H11","L6H0","L6H1","L6H2","L6H3","L6H4","L6H5","L6H6","L6H7","L6H8","L6H9","L6H10","L6H11","L7H0","L7H1","L7H2","L7H3","L7H4","L7H5","L7H6","L7H7","L7H8","L7H9","L7H10","L7H11","L8H0","L8H1","L8H2","L8H3","L8H4","L8H5","L8H6","L8H7","L8H8","L8H9","L8H10","L8H11","L9H0","L9H1","L9H2","L9H3","L9H4","L9H5","L9H6","L9H7","L9H8","L9H9","L9H10","L9H11","L10H0","L10H1","L10H2","L10H3","L10H4","L10H5","L10H6","L10H7","L10H8","L10H9","L10H10","L10H11","L11H0","L11H1","L11H2","L11H3","L11H4","L11H5","L11H6","L11H7","L11H8","L11H9","L11H10","L11H11"],"legendgroup":"","marker":{"color":"#636efa","symbol":"circle"},"mode":"markers","name":"","orientation":"v","showlegend":false,"x":[0.8356367349624634,0.5853534936904907,0.5105841159820557,0.7843377590179443,0.8644161224365234,0.7026589512825012,0.8969924449920654,0.5868823528289795,0.42486509680747986,-0.16337518393993378,0.46268585324287415,0.276053786277771,-0.052920110523700714,-0.31773144006729126,-0.4810579717159271,-0.7838066220283508,-0.6360211968421936,-0.77586829662323,0.9681803584098816,0.8119116425514221,-0.7510464787483215,-0.6878445744514465,-0.6429887413978577,-0.9985856413841248,-0.6598325967788696,0.9152501821517944,0.5461500883102417,-0.48743969202041626,0.7720563411712646,-0.7541061639785767,-0.8472450971603394,-0.6948987245559692,-0.1557510495185852,0.2444225549697876,-0.9106622338294983,-0.9439151287078857,0.648689866065979,-0.5592910647392273,0.5935593247413635,0.7843040823936462,-0.8150346279144287,0.613004744052887,0.16785882413387299,0.35195910930633545,-0.6837262511253357,0.2223764955997467,-0.9929219484329224,0.6535818576812744,0.5740953087806702,0.3640134036540985,0.09609051048755646,0.9359624981880188,-0.9228776097297668,0.6191076636314392,-0.33572638034820557,-0.998464822769165,0.6448632478713989,0.8468661308288574,-0.7557656764984131,0.9527971744537354,0.7326544523239136,0.5324169397354126,0.9732670187950134,0.7239246964454651,0.2553895115852356,0.8158416152000427,0.665579080581665,0.9287100434303284,-0.5660436153411865,-0.8908745050430298,0.9834234118461609,-0.9981179237365723,0.9698692560195923,0.7439671754837036,-0.35639333724975586,0.6022987365722656,0.9708116054534912,-0.9278275966644287,-0.9962316155433655,0.834520697593689,0.9714328050613403,0.8158544898033142,0.5902575850486755,0.8199343681335449,0.9820227026939392,0.9859329462051392,0.5152460932731628,-0.5610517263412476,0.9663666486740112,0.9495157599449158,-0.5204814076423645,0.3104752004146576,0.9859083890914917,0.7797460556030273,0.6738530397415161,0.39197394251823425,-0.906204104423523,0.11750960350036621,0.8077874779701233,0.416930615901947,0.9829014539718628,0.9902300834655762,0.7847104668617249,0.9945629835128784,0.9868024587631226,-0.26804426312446594,-0.9908867478370667,0.745792806148529,0.9906190633773804,-0.18231146037578583,0.9757838249206543,0.9986750483512878,0.2544330954551697,-0.954406201839447,0.586924135684967,-0.23537978529930115,0.9550502896308899,0.2551196813583374,0.9929869771003723,0.0905260294675827,0.9707273244857788,0.6956090927124023,0.6280022263526917,0.7902868390083313,0.9343841671943665,0.9895793795585632,0.9436282515525818,-0.10834990441799164,-0.3431110680103302,0.9986709952354431,0.508673906326294,0.9949509501457214,0.8283132910728455,0.9432437419891357,0.9491764903068542,0.9995353817939758,0.5712320804595947,0.8055236339569092,0.6781865954399109,0.8272573351860046,0.8314797282218933,0.8778655529022217,0.9944959282875061,0.997386634349823],"xaxis":"x","y":[0.7775008678436279,0.35272663831710815,0.25961872935295105,0.6670258045196533,0.838425874710083,0.5584433078765869,0.844474732875824,0.41379114985466003,0.244889497756958,0.02815753035247326,0.3584096431732178,0.16288283467292786,-0.4541913866996765,-0.6529324650764465,-0.5484569668769836,-0.7990368604660034,-0.7736426591873169,-0.8522582650184631,0.977432370185852,0.6626251935958862,-0.7303223609924316,-0.7007019519805908,-0.6946623921394348,-0.9996723532676697,-0.7837162613868713,0.8967758417129517,0.4750956892967224,-0.6671973466873169,0.7881463170051575,-0.8547748923301697,-0.9054183959960938,-0.5749384760856628,-0.321751207113266,-0.0285941194742918,-0.9247617125511169,-0.9699268937110901,0.5864036083221436,-0.76143479347229,0.5971695184707642,0.7854391932487488,-0.8788884878158569,0.3908745348453522,0.0447387658059597,0.1102800965309143,-0.8169987201690674,0.2212953418493271,-0.9939578771591187,0.5774401426315308,0.5254791378974915,0.30490121245384216,-0.10729170590639114,0.9433152675628662,-0.9314427971839905,0.5273631811141968,-0.4264712929725647,-0.9984429478645325,0.5296757817268372,0.8604294657707214,-0.8895052075386047,0.9556970000267029,0.6629188656806946,0.4295697808265686,0.9736858010292053,0.655548095703125,0.12201863527297974,0.7442769408226013,0.5037954449653625,0.95253586769104,-0.6507166624069214,-0.9316278100013733,0.9791510701179504,-0.9972586035728455,0.9613031148910522,0.7501779794692993,-0.3806658983230591,0.6429785490036011,0.9557769298553467,-0.9428837895393372,-0.9948079586029053,0.7852989435195923,0.9657301306724548,0.7073014974594116,0.36872273683547974,0.8128010034561157,0.9659482836723328,0.9730120301246643,0.31900620460510254,-0.30290529131889343,0.9790952801704407,0.9357922673225403,-0.5550314784049988,-0.0054661668837070465,0.986777663230896,0.8249568343162537,0.5664296746253967,0.1000528484582901,-0.9464486837387085,-0.25471991300582886,0.6522327065467834,0.14152583479881287,0.9884141683578491,0.9860584735870361,0.6949271559715271,0.9901811480522156,0.9791203141212463,-0.23595543205738068,-0.982071042060852,0.6506688594818115,0.9895945191383362,-0.291781485080719,0.9714024662971497,0.9951602220535278,0.18783727288246155,-0.9460937976837158,0.4780191481113434,-0.2489192634820938,0.9437099099159241,0.11866225302219391,0.9941242933273315,-0.38088199496269226,0.9564487338066101,0.5542722344398499,0.42118069529533386,0.6628788113594055,0.8659593462944031,0.9937117695808411,0.9069075584411621,0.3981107473373413,-0.4134218096733093,0.9971914887428284,0.3459664583206177,0.9938657283782959,0.5891268253326416,0.9313738942146301,0.9268401861190796,0.9993563890457153,0.6227542161941528,0.8463947772979736,0.6584343910217285,0.8423123955726624,0.2978499233722687,0.8728678822517395,0.9963143467903137,0.9867526292800903],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"Full OV Copying Score"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"OV Copying Score"}},"legend":{"tracegroupgap":0},"title":{"text":"OV Copying Score for each head in GPT-2 Small"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="916543d2-faa6-4038-b529-7295899da0ca" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("916543d2-faa6-4038-b529-7295899da0ca")) {                    Plotly.newPlot(                        "916543d2-faa6-4038-b529-7295899da0ca",                        [{"hovertemplate":"\u003cb\u003e%{hovertext}\u003c\u002fb\u003e\u003cbr\u003e\u003cbr\u003eFull OV Copying Score=%{x}\u003cbr\u003eOV Copying Score=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","hovertext":["L0H0","L0H1","L0H2","L0H3","L0H4","L0H5","L0H6","L0H7","L0H8","L0H9","L0H10","L0H11","L1H0","L1H1","L1H2","L1H3","L1H4","L1H5","L1H6","L1H7","L1H8","L1H9","L1H10","L1H11","L2H0","L2H1","L2H2","L2H3","L2H4","L2H5","L2H6","L2H7","L2H8","L2H9","L2H10","L2H11","L3H0","L3H1","L3H2","L3H3","L3H4","L3H5","L3H6","L3H7","L3H8","L3H9","L3H10","L3H11","L4H0","L4H1","L4H2","L4H3","L4H4","L4H5","L4H6","L4H7","L4H8","L4H9","L4H10","L4H11","L5H0","L5H1","L5H2","L5H3","L5H4","L5H5","L5H6","L5H7","L5H8","L5H9","L5H10","L5H11","L6H0","L6H1","L6H2","L6H3","L6H4","L6H5","L6H6","L6H7","L6H8","L6H9","L6H10","L6H11","L7H0","L7H1","L7H2","L7H3","L7H4","L7H5","L7H6","L7H7","L7H8","L7H9","L7H10","L7H11","L8H0","L8H1","L8H2","L8H3","L8H4","L8H5","L8H6","L8H7","L8H8","L8H9","L8H10","L8H11","L9H0","L9H1","L9H2","L9H3","L9H4","L9H5","L9H6","L9H7","L9H8","L9H9","L9H10","L9H11","L10H0","L10H1","L10H2","L10H3","L10H4","L10H5","L10H6","L10H7","L10H8","L10H9","L10H10","L10H11","L11H0","L11H1","L11H2","L11H3","L11H4","L11H5","L11H6","L11H7","L11H8","L11H9","L11H10","L11H11"],"legendgroup":"","marker":{"color":"#636efa","symbol":"circle"},"mode":"markers","name":"","orientation":"v","showlegend":false,"x":[0.8356367349624634,0.5853534936904907,0.5105841159820557,0.7843377590179443,0.8644161224365234,0.7026589512825012,0.8969924449920654,0.5868823528289795,0.42486509680747986,-0.16337518393993378,0.46268585324287415,0.276053786277771,-0.052920110523700714,-0.31773144006729126,-0.4810579717159271,-0.7838066220283508,-0.6360211968421936,-0.77586829662323,0.9681803584098816,0.8119116425514221,-0.7510464787483215,-0.6878445744514465,-0.6429887413978577,-0.9985856413841248,-0.6598325967788696,0.9152501821517944,0.5461500883102417,-0.48743969202041626,0.7720563411712646,-0.7541061639785767,-0.8472450971603394,-0.6948987245559692,-0.1557510495185852,0.2444225549697876,-0.9106622338294983,-0.9439151287078857,0.648689866065979,-0.5592910647392273,0.5935593247413635,0.7843040823936462,-0.8150346279144287,0.613004744052887,0.16785882413387299,0.35195910930633545,-0.6837262511253357,0.2223764955997467,-0.9929219484329224,0.6535818576812744,0.5740953087806702,0.3640134036540985,0.09609051048755646,0.9359624981880188,-0.9228776097297668,0.6191076636314392,-0.33572638034820557,-0.998464822769165,0.6448632478713989,0.8468661308288574,-0.7557656764984131,0.9527971744537354,0.7326544523239136,0.5324169397354126,0.9732670187950134,0.7239246964454651,0.2553895115852356,0.8158416152000427,0.665579080581665,0.9287100434303284,-0.5660436153411865,-0.8908745050430298,0.9834234118461609,-0.9981179237365723,0.9698692560195923,0.7439671754837036,-0.35639333724975586,0.6022987365722656,0.9708116054534912,-0.9278275966644287,-0.9962316155433655,0.834520697593689,0.9714328050613403,0.8158544898033142,0.5902575850486755,0.8199343681335449,0.9820227026939392,0.9859329462051392,0.5152460932731628,-0.5610517263412476,0.9663666486740112,0.9495157599449158,-0.5204814076423645,0.3104752004146576,0.9859083890914917,0.7797460556030273,0.6738530397415161,0.39197394251823425,-0.906204104423523,0.11750960350036621,0.8077874779701233,0.416930615901947,0.9829014539718628,0.9902300834655762,0.7847104668617249,0.9945629835128784,0.9868024587631226,-0.26804426312446594,-0.9908867478370667,0.745792806148529,0.9906190633773804,-0.18231146037578583,0.9757838249206543,0.9986750483512878,0.2544330954551697,-0.954406201839447,0.586924135684967,-0.23537978529930115,0.9550502896308899,0.2551196813583374,0.9929869771003723,0.0905260294675827,0.9707273244857788,0.6956090927124023,0.6280022263526917,0.7902868390083313,0.9343841671943665,0.9895793795585632,0.9436282515525818,-0.10834990441799164,-0.3431110680103302,0.9986709952354431,0.508673906326294,0.9949509501457214,0.8283132910728455,0.9432437419891357,0.9491764903068542,0.9995353817939758,0.5712320804595947,0.8055236339569092,0.6781865954399109,0.8272573351860046,0.8314797282218933,0.8778655529022217,0.9944959282875061,0.997386634349823],"xaxis":"x","y":[0.7775008678436279,0.35272663831710815,0.25961872935295105,0.6670258045196533,0.838425874710083,0.5584433078765869,0.844474732875824,0.41379114985466003,0.244889497756958,0.02815753035247326,0.3584096431732178,0.16288283467292786,-0.4541913866996765,-0.6529324650764465,-0.5484569668769836,-0.7990368604660034,-0.7736426591873169,-0.8522582650184631,0.977432370185852,0.6626251935958862,-0.7303223609924316,-0.7007019519805908,-0.6946623921394348,-0.9996723532676697,-0.7837162613868713,0.8967758417129517,0.4750956892967224,-0.6671973466873169,0.7881463170051575,-0.8547748923301697,-0.9054183959960938,-0.5749384760856628,-0.321751207113266,-0.0285941194742918,-0.9247617125511169,-0.9699268937110901,0.5864036083221436,-0.76143479347229,0.5971695184707642,0.7854391932487488,-0.8788884878158569,0.3908745348453522,0.0447387658059597,0.1102800965309143,-0.8169987201690674,0.2212953418493271,-0.9939578771591187,0.5774401426315308,0.5254791378974915,0.30490121245384216,-0.10729170590639114,0.9433152675628662,-0.9314427971839905,0.5273631811141968,-0.4264712929725647,-0.9984429478645325,0.5296757817268372,0.8604294657707214,-0.8895052075386047,0.9556970000267029,0.6629188656806946,0.4295697808265686,0.9736858010292053,0.655548095703125,0.12201863527297974,0.7442769408226013,0.5037954449653625,0.95253586769104,-0.6507166624069214,-0.9316278100013733,0.9791510701179504,-0.9972586035728455,0.9613031148910522,0.7501779794692993,-0.3806658983230591,0.6429785490036011,0.9557769298553467,-0.9428837895393372,-0.9948079586029053,0.7852989435195923,0.9657301306724548,0.7073014974594116,0.36872273683547974,0.8128010034561157,0.9659482836723328,0.9730120301246643,0.31900620460510254,-0.30290529131889343,0.9790952801704407,0.9357922673225403,-0.5550314784049988,-0.0054661668837070465,0.986777663230896,0.8249568343162537,0.5664296746253967,0.1000528484582901,-0.9464486837387085,-0.25471991300582886,0.6522327065467834,0.14152583479881287,0.9884141683578491,0.9860584735870361,0.6949271559715271,0.9901811480522156,0.9791203141212463,-0.23595543205738068,-0.982071042060852,0.6506688594818115,0.9895945191383362,-0.291781485080719,0.9714024662971497,0.9951602220535278,0.18783727288246155,-0.9460937976837158,0.4780191481113434,-0.2489192634820938,0.9437099099159241,0.11866225302219391,0.9941242933273315,-0.38088199496269226,0.9564487338066101,0.5542722344398499,0.42118069529533386,0.6628788113594055,0.8659593462944031,0.9937117695808411,0.9069075584411621,0.3981107473373413,-0.4134218096733093,0.9971914887428284,0.3459664583206177,0.9938657283782959,0.5891268253326416,0.9313738942146301,0.9268401861190796,0.9993563890457153,0.6227542161941528,0.8463947772979736,0.6584343910217285,0.8423123955726624,0.2978499233722687,0.8728678822517395,0.9963143467903137,0.9867526292800903],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"Full OV Copying Score"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"OV Copying Score"}},"legend":{"tracegroupgap":0},"title":{"text":"OV Copying Score for each head in GPT-2 Small"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('3c8c77e9-6054-4886-85c5-8868e8e7c01c');
+var gd = document.getElementById('916543d2-faa6-4038-b529-7295899da0ca');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2199,7 +2184,7 @@ <h2>Generating Text<a class="headerlink" href="#Generating-Text" title="Permalin
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "ef67630185584553867488aa7abe9fa9"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "428a330dae074773aad72f7de9f6b219"}</script></div>
 </div>
 <div class="nboutput nblast docutils container">
 <div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[54]:
@@ -2357,9 +2342,9 @@ <h2>Loading Pre-Trained Checkpoints<a class="headerlink" href="#Loading-Pre-Trai
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="76053182-4b0b-44d5-9e6f-dc68cfcabb60" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("76053182-4b0b-44d5-9e6f-dc68cfcabb60")) {                    Plotly.newPlot(                        "76053182-4b0b-44d5-9e6f-dc68cfcabb60",                        [{"hovertemplate":"variable=0\u003cbr\u003eindex=%{x}\u003cbr\u003evalue=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"0","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines+markers","name":"0","orientation":"v","showlegend":true,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162],"xaxis":"x","y":[262144,2621440,4718592,7077888,9175040,11272192,13631488,15728640,18087936,20185088,22282240,33292288,44302336,55312384,66322432,77332480,88342528,99352576,110362624,121372672,132382720,143392768,154402816,165412864,176422912,187432960,198443008,209453056,220463104,264503296,308281344,352321536,396361728,440401920,484442112,528482304,572522496,616300544,660340736,704380928,748421120,792461312,836501504,880279552,924319744,968359936,1012400128,1056440320,1100480512,1144520704,1188298752,1232338944,1276379136,1320419328,1364459520,1408499712,1452277760,1496317952,1540358144,1584398336,1628438528,1672478720,1716518912,1760296960,1804337152,1848377344,1892417536,1936457728,1980497920,2024275968,2068316160,2112356352,2156396544,2200436736,2420375552,2640314368,2860515328,3080454144,3300392960,3520331776,3740270592,3960471552,4180410368,4400349184,4620288000,4840488960,5060427776,5280366592,5500305408,5720506368,5940445184,6160384000,6380322816,6600523776,6820462592,7040401408,7260340224,7480279040,7700480000,7920418816,8140357632,8360296448,8580497408,8800436224,9020375040,9240313856,9460514816,9680453632,9900392448,10120331264,10340270080,10560471040,10780409856,11000348672,11220287488,11440488448,11660427264,11880366080,12100304896,12320505856,12540444672,12760383488,12980322304,13200523264,13420462080,13640400896,13860339712,14080278528,14300479488,14520418304,14740357120,14960295936,15180496896,15400435712,15620374528,15840313344,16060514304,16280453120,16500391936,16720330752,16940269568,17160470528,17380409344,17600348160,17820286976,18040487936,18260426752,18480365568,18700304384,18920505344,19140444160,19360382976,19580321792,19800522752,20020461568,20240400384,20460339200,20680278016,20900478976,21120417792,21340356608,21560295424,21780496384],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"index"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"value"},"type":"log"},"legend":{"title":{"text":"variable"},"tracegroupgap":0},"title":{"text":"Checkpoint Values for attn-only-2l (Log scale)"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="5b380b3b-e660-43ec-a723-02466b223aae" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("5b380b3b-e660-43ec-a723-02466b223aae")) {                    Plotly.newPlot(                        "5b380b3b-e660-43ec-a723-02466b223aae",                        [{"hovertemplate":"variable=0\u003cbr\u003eindex=%{x}\u003cbr\u003evalue=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"0","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines+markers","name":"0","orientation":"v","showlegend":true,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162],"xaxis":"x","y":[262144,2621440,4718592,7077888,9175040,11272192,13631488,15728640,18087936,20185088,22282240,33292288,44302336,55312384,66322432,77332480,88342528,99352576,110362624,121372672,132382720,143392768,154402816,165412864,176422912,187432960,198443008,209453056,220463104,264503296,308281344,352321536,396361728,440401920,484442112,528482304,572522496,616300544,660340736,704380928,748421120,792461312,836501504,880279552,924319744,968359936,1012400128,1056440320,1100480512,1144520704,1188298752,1232338944,1276379136,1320419328,1364459520,1408499712,1452277760,1496317952,1540358144,1584398336,1628438528,1672478720,1716518912,1760296960,1804337152,1848377344,1892417536,1936457728,1980497920,2024275968,2068316160,2112356352,2156396544,2200436736,2420375552,2640314368,2860515328,3080454144,3300392960,3520331776,3740270592,3960471552,4180410368,4400349184,4620288000,4840488960,5060427776,5280366592,5500305408,5720506368,5940445184,6160384000,6380322816,6600523776,6820462592,7040401408,7260340224,7480279040,7700480000,7920418816,8140357632,8360296448,8580497408,8800436224,9020375040,9240313856,9460514816,9680453632,9900392448,10120331264,10340270080,10560471040,10780409856,11000348672,11220287488,11440488448,11660427264,11880366080,12100304896,12320505856,12540444672,12760383488,12980322304,13200523264,13420462080,13640400896,13860339712,14080278528,14300479488,14520418304,14740357120,14960295936,15180496896,15400435712,15620374528,15840313344,16060514304,16280453120,16500391936,16720330752,16940269568,17160470528,17380409344,17600348160,17820286976,18040487936,18260426752,18480365568,18700304384,18920505344,19140444160,19360382976,19580321792,19800522752,20020461568,20240400384,20460339200,20680278016,20900478976,21120417792,21340356608,21560295424,21780496384],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"index"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"value"},"type":"log"},"legend":{"title":{"text":"variable"},"tracegroupgap":0},"title":{"text":"Checkpoint Values for attn-only-2l (Log scale)"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('76053182-4b0b-44d5-9e6f-dc68cfcabb60');
+var gd = document.getElementById('5b380b3b-e660-43ec-a723-02466b223aae');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2393,9 +2378,9 @@ <h2>Loading Pre-Trained Checkpoints<a class="headerlink" href="#Loading-Pre-Trai
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="22829199-cf37-49cf-8817-a42bc6f79448" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("22829199-cf37-49cf-8817-a42bc6f79448")) {                    Plotly.newPlot(                        "22829199-cf37-49cf-8817-a42bc6f79448",                        [{"hovertemplate":"variable=0\u003cbr\u003eindex=%{x}\u003cbr\u003evalue=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"0","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines+markers","name":"0","orientation":"v","showlegend":true,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162],"xaxis":"x","y":[196608,3342336,6291456,9240576,12386304,15335424,18284544,21233664,24379392,27328512,30277632,45219840,60358656,75300864,90243072,105381888,120324096,135266304,150208512,165347328,180289536,195231744,210370560,225312768,240254976,255197184,270336000,285278208,300220416,360382464,420347904,480313344,540278784,600244224,660209664,720371712,780337152,840302592,900268032,960233472,1020198912,1080360960,1140326400,1200291840,1260257280,1320222720,1380384768,1440350208,1500315648,1560281088,1620246528,1680211968,1740374016,1800339456,1860304896,1920270336,1980235776,2040201216,2100363264,2160328704,2220294144,2280259584,2340225024,2400387072,2460352512,2520317952,2580283392,2640248832,2700214272,2760376320,2820341760,2880307200,2940272640,3000238080,3300261888,3600285696,3900309504,4200333312,4500357120,4800380928,5100208128,5400231936,5700255744,6000279552,6300303360,6600327168,6900350976,7200374784,7500201984,7800225792,8100249600,8400273408,8700297216,9000321024,9300344832,9600368640,9900392448,10200219648,10500243456,10800267264,11100291072,11400314880,11700338688,12000362496,12300386304,12600213504,12900237312,13200261120,13500284928,13800308736,14100332544,14400356352,14700380160,15000207360,15300231168,15600254976,15900278784,16200302592,16500326400,16800350208,17100374016,17400201216,17700225024,18000248832,18300272640,18600296448,18900320256,19200344064,19500367872,19800391680,20100218880,20400242688,20700266496,21000290304,21300314112,21600337920,21900361728,22200385536,22500212736,22800236544,23100260352,23400284160,23700307968,24000331776,24300355584,24600379392,24900206592,25200230400,25500254208,25800278016,26100301824,26400325632,26700349440,27000373248,27300200448,27600224256,27900248064,28200271872,28500295680,28800319488,29100343296,29400367104,29700390912],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"index"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"value"},"type":"log"},"legend":{"title":{"text":"variable"},"tracegroupgap":0},"title":{"text":"Checkpoint Values for solu-12l (Log scale)"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="2e1b7810-7383-4f28-8e08-f69e548f4ae1" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("2e1b7810-7383-4f28-8e08-f69e548f4ae1")) {                    Plotly.newPlot(                        "2e1b7810-7383-4f28-8e08-f69e548f4ae1",                        [{"hovertemplate":"variable=0\u003cbr\u003eindex=%{x}\u003cbr\u003evalue=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"0","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines+markers","name":"0","orientation":"v","showlegend":true,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162],"xaxis":"x","y":[196608,3342336,6291456,9240576,12386304,15335424,18284544,21233664,24379392,27328512,30277632,45219840,60358656,75300864,90243072,105381888,120324096,135266304,150208512,165347328,180289536,195231744,210370560,225312768,240254976,255197184,270336000,285278208,300220416,360382464,420347904,480313344,540278784,600244224,660209664,720371712,780337152,840302592,900268032,960233472,1020198912,1080360960,1140326400,1200291840,1260257280,1320222720,1380384768,1440350208,1500315648,1560281088,1620246528,1680211968,1740374016,1800339456,1860304896,1920270336,1980235776,2040201216,2100363264,2160328704,2220294144,2280259584,2340225024,2400387072,2460352512,2520317952,2580283392,2640248832,2700214272,2760376320,2820341760,2880307200,2940272640,3000238080,3300261888,3600285696,3900309504,4200333312,4500357120,4800380928,5100208128,5400231936,5700255744,6000279552,6300303360,6600327168,6900350976,7200374784,7500201984,7800225792,8100249600,8400273408,8700297216,9000321024,9300344832,9600368640,9900392448,10200219648,10500243456,10800267264,11100291072,11400314880,11700338688,12000362496,12300386304,12600213504,12900237312,13200261120,13500284928,13800308736,14100332544,14400356352,14700380160,15000207360,15300231168,15600254976,15900278784,16200302592,16500326400,16800350208,17100374016,17400201216,17700225024,18000248832,18300272640,18600296448,18900320256,19200344064,19500367872,19800391680,20100218880,20400242688,20700266496,21000290304,21300314112,21600337920,21900361728,22200385536,22500212736,22800236544,23100260352,23400284160,23700307968,24000331776,24300355584,24600379392,24900206592,25200230400,25500254208,25800278016,26100301824,26400325632,26700349440,27000373248,27300200448,27600224256,27900248064,28200271872,28500295680,28800319488,29100343296,29400367104,29700390912],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"index"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"value"},"type":"log"},"legend":{"title":{"text":"variable"},"tracegroupgap":0},"title":{"text":"Checkpoint Values for solu-12l (Log scale)"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('22829199-cf37-49cf-8817-a42bc6f79448');
+var gd = document.getElementById('2e1b7810-7383-4f28-8e08-f69e548f4ae1');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2429,9 +2414,9 @@ <h2>Loading Pre-Trained Checkpoints<a class="headerlink" href="#Loading-Pre-Trai
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="86fd24b0-a205-4e6c-8105-f79fcf57c539" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("86fd24b0-a205-4e6c-8105-f79fcf57c539")) {                    Plotly.newPlot(                        "86fd24b0-a205-4e6c-8105-f79fcf57c539",                        [{"hovertemplate":"variable=0\u003cbr\u003eindex=%{x}\u003cbr\u003evalue=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"0","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines+markers","name":"0","orientation":"v","showlegend":true,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536,537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603,604,605,606,607,608],"xaxis":"x","y":[0,10,20,30,40,50,60,70,80,90,100,150,200,250,300,350,400,450,500,550,600,650,700,750,800,850,900,950,1000,1050,1100,1150,1200,1250,1300,1350,1400,1450,1500,1550,1600,1650,1700,1750,1800,1850,1900,1950,2000,2100,2200,2300,2400,2500,2600,2700,2800,2900,3000,3100,3200,3300,3400,3500,3600,3700,3800,3900,4000,4100,4200,4300,4400,4500,4600,4700,4800,4900,5000,5100,5200,5300,5400,5500,5600,5700,5800,5900,6000,6100,6200,6300,6400,6500,6600,6700,6800,6900,7000,7100,7200,7300,7400,7500,7600,7700,7800,7900,8000,8100,8200,8300,8400,8500,8600,8700,8800,8900,9000,9100,9200,9300,9400,9500,9600,9700,9800,9900,10000,10100,10200,10300,10400,10500,10600,10700,10800,10900,11000,11100,11200,11300,11400,11500,11600,11700,11800,11900,12000,12100,12200,12300,12400,12500,12600,12700,12800,12900,13000,13100,13200,13300,13400,13500,13600,13700,13800,13900,14000,14100,14200,14300,14400,14500,14600,14700,14800,14900,15000,15100,15200,15300,15400,15500,15600,15700,15800,15900,16000,16100,16200,16300,16400,16500,16600,16700,16800,16900,17000,17100,17200,17300,17400,17500,17600,17700,17800,17900,18000,18100,18200,18300,18400,18500,18600,18700,18800,18900,19000,19100,19200,19300,19400,19500,19600,19700,19800,19900,20000,21000,22000,23000,24000,25000,26000,27000,28000,29000,30000,31000,32000,33000,34000,35000,36000,37000,38000,39000,40000,41000,42000,43000,44000,45000,46000,47000,48000,49000,50000,51000,52000,53000,54000,55000,56000,57000,58000,59000,60000,61000,62000,63000,64000,65000,66000,67000,68000,69000,70000,71000,72000,73000,74000,75000,76000,77000,78000,79000,80000,81000,82000,83000,84000,85000,86000,87000,88000,89000,90000,91000,92000,93000,94000,95000,96000,97000,98000,99000,100000,101000,102000,103000,104000,105000,106000,107000,108000,109000,110000,111000,112000,113000,114000,115000,116000,117000,118000,119000,120000,121000,122000,123000,124000,125000,126000,127000,128000,129000,130000,131000,132000,133000,134000,135000,136000,137000,138000,139000,140000,141000,142000,143000,144000,145000,146000,147000,148000,149000,150000,151000,152000,153000,154000,155000,156000,157000,158000,159000,160000,161000,162000,163000,164000,165000,166000,167000,168000,169000,170000,171000,172000,173000,174000,175000,176000,177000,178000,179000,180000,181000,182000,183000,184000,185000,186000,187000,188000,189000,190000,191000,192000,193000,194000,195000,196000,197000,198000,199000,200000,201000,202000,203000,204000,205000,206000,207000,208000,209000,210000,211000,212000,213000,214000,215000,216000,217000,218000,219000,220000,221000,222000,223000,224000,225000,226000,227000,228000,229000,230000,231000,232000,233000,234000,235000,236000,237000,238000,239000,240000,241000,242000,243000,244000,245000,246000,247000,248000,249000,250000,251000,252000,253000,254000,255000,256000,257000,258000,259000,260000,261000,262000,263000,264000,265000,266000,267000,268000,269000,270000,271000,272000,273000,274000,275000,276000,277000,278000,279000,280000,281000,282000,283000,284000,285000,286000,287000,288000,289000,290000,291000,292000,293000,294000,295000,296000,297000,298000,299000,300000,301000,302000,303000,304000,305000,306000,307000,308000,309000,310000,311000,312000,313000,314000,315000,316000,317000,318000,319000,320000,321000,322000,323000,324000,325000,326000,327000,328000,329000,330000,331000,332000,333000,334000,335000,336000,337000,338000,339000,340000,341000,342000,343000,344000,345000,346000,347000,348000,349000,350000,351000,352000,353000,354000,355000,356000,357000,358000,359000,360000,361000,362000,363000,364000,365000,366000,367000,368000,369000,370000,371000,372000,373000,374000,375000,376000,377000,378000,379000,380000,381000,382000,383000,384000,385000,386000,387000,388000,389000,390000,391000,392000,393000,394000,395000,396000,397000,398000,399000,400000],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"index"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"value"},"type":"log"},"legend":{"title":{"text":"variable"},"tracegroupgap":0},"title":{"text":"Checkpoint Values for stanford-gpt2-small-a (Log scale)"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="81083222-8fc2-4750-982b-6c7df91a2cfc" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("81083222-8fc2-4750-982b-6c7df91a2cfc")) {                    Plotly.newPlot(                        "81083222-8fc2-4750-982b-6c7df91a2cfc",                        [{"hovertemplate":"variable=0\u003cbr\u003eindex=%{x}\u003cbr\u003evalue=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"0","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines+markers","name":"0","orientation":"v","showlegend":true,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536,537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603,604,605,606,607,608],"xaxis":"x","y":[0,10,20,30,40,50,60,70,80,90,100,150,200,250,300,350,400,450,500,550,600,650,700,750,800,850,900,950,1000,1050,1100,1150,1200,1250,1300,1350,1400,1450,1500,1550,1600,1650,1700,1750,1800,1850,1900,1950,2000,2100,2200,2300,2400,2500,2600,2700,2800,2900,3000,3100,3200,3300,3400,3500,3600,3700,3800,3900,4000,4100,4200,4300,4400,4500,4600,4700,4800,4900,5000,5100,5200,5300,5400,5500,5600,5700,5800,5900,6000,6100,6200,6300,6400,6500,6600,6700,6800,6900,7000,7100,7200,7300,7400,7500,7600,7700,7800,7900,8000,8100,8200,8300,8400,8500,8600,8700,8800,8900,9000,9100,9200,9300,9400,9500,9600,9700,9800,9900,10000,10100,10200,10300,10400,10500,10600,10700,10800,10900,11000,11100,11200,11300,11400,11500,11600,11700,11800,11900,12000,12100,12200,12300,12400,12500,12600,12700,12800,12900,13000,13100,13200,13300,13400,13500,13600,13700,13800,13900,14000,14100,14200,14300,14400,14500,14600,14700,14800,14900,15000,15100,15200,15300,15400,15500,15600,15700,15800,15900,16000,16100,16200,16300,16400,16500,16600,16700,16800,16900,17000,17100,17200,17300,17400,17500,17600,17700,17800,17900,18000,18100,18200,18300,18400,18500,18600,18700,18800,18900,19000,19100,19200,19300,19400,19500,19600,19700,19800,19900,20000,21000,22000,23000,24000,25000,26000,27000,28000,29000,30000,31000,32000,33000,34000,35000,36000,37000,38000,39000,40000,41000,42000,43000,44000,45000,46000,47000,48000,49000,50000,51000,52000,53000,54000,55000,56000,57000,58000,59000,60000,61000,62000,63000,64000,65000,66000,67000,68000,69000,70000,71000,72000,73000,74000,75000,76000,77000,78000,79000,80000,81000,82000,83000,84000,85000,86000,87000,88000,89000,90000,91000,92000,93000,94000,95000,96000,97000,98000,99000,100000,101000,102000,103000,104000,105000,106000,107000,108000,109000,110000,111000,112000,113000,114000,115000,116000,117000,118000,119000,120000,121000,122000,123000,124000,125000,126000,127000,128000,129000,130000,131000,132000,133000,134000,135000,136000,137000,138000,139000,140000,141000,142000,143000,144000,145000,146000,147000,148000,149000,150000,151000,152000,153000,154000,155000,156000,157000,158000,159000,160000,161000,162000,163000,164000,165000,166000,167000,168000,169000,170000,171000,172000,173000,174000,175000,176000,177000,178000,179000,180000,181000,182000,183000,184000,185000,186000,187000,188000,189000,190000,191000,192000,193000,194000,195000,196000,197000,198000,199000,200000,201000,202000,203000,204000,205000,206000,207000,208000,209000,210000,211000,212000,213000,214000,215000,216000,217000,218000,219000,220000,221000,222000,223000,224000,225000,226000,227000,228000,229000,230000,231000,232000,233000,234000,235000,236000,237000,238000,239000,240000,241000,242000,243000,244000,245000,246000,247000,248000,249000,250000,251000,252000,253000,254000,255000,256000,257000,258000,259000,260000,261000,262000,263000,264000,265000,266000,267000,268000,269000,270000,271000,272000,273000,274000,275000,276000,277000,278000,279000,280000,281000,282000,283000,284000,285000,286000,287000,288000,289000,290000,291000,292000,293000,294000,295000,296000,297000,298000,299000,300000,301000,302000,303000,304000,305000,306000,307000,308000,309000,310000,311000,312000,313000,314000,315000,316000,317000,318000,319000,320000,321000,322000,323000,324000,325000,326000,327000,328000,329000,330000,331000,332000,333000,334000,335000,336000,337000,338000,339000,340000,341000,342000,343000,344000,345000,346000,347000,348000,349000,350000,351000,352000,353000,354000,355000,356000,357000,358000,359000,360000,361000,362000,363000,364000,365000,366000,367000,368000,369000,370000,371000,372000,373000,374000,375000,376000,377000,378000,379000,380000,381000,382000,383000,384000,385000,386000,387000,388000,389000,390000,391000,392000,393000,394000,395000,396000,397000,398000,399000,400000],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"index"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"value"},"type":"log"},"legend":{"title":{"text":"variable"},"tracegroupgap":0},"title":{"text":"Checkpoint Values for stanford-gpt2-small-a (Log scale)"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('86fd24b0-a205-4e6c-8105-f79fcf57c539');
+var gd = document.getElementById('81083222-8fc2-4750-982b-6c7df91a2cfc');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2465,9 +2450,9 @@ <h2>Loading Pre-Trained Checkpoints<a class="headerlink" href="#Loading-Pre-Trai
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="4a6e81dc-ddca-460f-873f-0a213188ab89" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("4a6e81dc-ddca-460f-873f-0a213188ab89")) {                    Plotly.newPlot(                        "4a6e81dc-ddca-460f-873f-0a213188ab89",                        [{"hovertemplate":"variable=0\u003cbr\u003eindex=%{x}\u003cbr\u003evalue=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"0","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines+markers","name":"0","orientation":"v","showlegend":true,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49],"xaxis":"x","y":[832,1664,2496,3328,4160,4992,5824,6656,7488,8320,9152,9984,10816,11648,12480,13312,14144,14976,15808,16640,17472,18304,19136,19968,20800,21632,22464,23296,24128,24960,25792,26624,27456,28288,29120,29952,30784,31616,32448,33280,34112,34944,35776,36608,37440,38272,39104,39936,40768,41600],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"index"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"value"}},"legend":{"title":{"text":"variable"},"tracegroupgap":0},"title":{"text":"Checkpoint Values for solu-1l-pile (Linear scale)"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="f160c230-ab5d-46fa-be53-58d80e50a0e7" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("f160c230-ab5d-46fa-be53-58d80e50a0e7")) {                    Plotly.newPlot(                        "f160c230-ab5d-46fa-be53-58d80e50a0e7",                        [{"hovertemplate":"variable=0\u003cbr\u003eindex=%{x}\u003cbr\u003evalue=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"0","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines+markers","name":"0","orientation":"v","showlegend":true,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49],"xaxis":"x","y":[832,1664,2496,3328,4160,4992,5824,6656,7488,8320,9152,9984,10816,11648,12480,13312,14144,14976,15808,16640,17472,18304,19136,19968,20800,21632,22464,23296,24128,24960,25792,26624,27456,28288,29120,29952,30784,31616,32448,33280,34112,34944,35776,36608,37440,38272,39104,39936,40768,41600],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"index"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"value"}},"legend":{"title":{"text":"variable"},"tracegroupgap":0},"title":{"text":"Checkpoint Values for solu-1l-pile (Linear scale)"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('4a6e81dc-ddca-460f-873f-0a213188ab89');
+var gd = document.getElementById('f160c230-ab5d-46fa-be53-58d80e50a0e7');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2501,9 +2486,9 @@ <h2>Loading Pre-Trained Checkpoints<a class="headerlink" href="#Loading-Pre-Trai
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="e2877398-410c-47b5-9f61-1fd97414d652" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("e2877398-410c-47b5-9f61-1fd97414d652")) {                    Plotly.newPlot(                        "e2877398-410c-47b5-9f61-1fd97414d652",                        [{"hovertemplate":"variable=0\u003cbr\u003eindex=%{x}\u003cbr\u003evalue=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"0","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines+markers","name":"0","orientation":"v","showlegend":true,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99],"xaxis":"x","y":[326,652,978,1304,1630,1956,2282,2608,2934,3260,3586,3912,4238,4564,4890,5216,5542,5868,6194,6520,6846,7172,7498,7824,8150,8476,8802,9128,9454,9780,10106,10432,10758,11084,11410,11736,12062,12388,12714,13040,13366,13692,14018,14344,14670,14996,15322,15648,15974,16300,16626,16952,17278,17604,17930,18256,18582,18908,19234,19560,19886,20212,20538,20864,21190,21516,21842,22168,22494,22820,23146,23472,23798,24124,24450,24776,25102,25428,25754,26080,26406,26732,27058,27384,27710,28036,28362,28688,29014,29340,29666,29992,30318,30644,30970,31296,31622,31948,32274,32600],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"index"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"value"}},"legend":{"title":{"text":"variable"},"tracegroupgap":0},"title":{"text":"Checkpoint Values for solu-6l-pile (Linear scale)"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="77531c7d-147c-41c7-88a0-f58896e92710" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("77531c7d-147c-41c7-88a0-f58896e92710")) {                    Plotly.newPlot(                        "77531c7d-147c-41c7-88a0-f58896e92710",                        [{"hovertemplate":"variable=0\u003cbr\u003eindex=%{x}\u003cbr\u003evalue=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"0","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines+markers","name":"0","orientation":"v","showlegend":true,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99],"xaxis":"x","y":[326,652,978,1304,1630,1956,2282,2608,2934,3260,3586,3912,4238,4564,4890,5216,5542,5868,6194,6520,6846,7172,7498,7824,8150,8476,8802,9128,9454,9780,10106,10432,10758,11084,11410,11736,12062,12388,12714,13040,13366,13692,14018,14344,14670,14996,15322,15648,15974,16300,16626,16952,17278,17604,17930,18256,18582,18908,19234,19560,19886,20212,20538,20864,21190,21516,21842,22168,22494,22820,23146,23472,23798,24124,24450,24776,25102,25428,25754,26080,26406,26732,27058,27384,27710,28036,28362,28688,29014,29340,29666,29992,30318,30644,30970,31296,31622,31948,32274,32600],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"index"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"value"}},"legend":{"title":{"text":"variable"},"tracegroupgap":0},"title":{"text":"Checkpoint Values for solu-6l-pile (Linear scale)"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('e2877398-410c-47b5-9f61-1fd97414d652');
+var gd = document.getElementById('77531c7d-147c-41c7-88a0-f58896e92710');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2588,9 +2573,9 @@ <h3>Example: Induction Head Phase Transition<a class="headerlink" href="#Example
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="e8c48c58-6618-4eb6-8790-24b50ffe4409" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("e8c48c58-6618-4eb6-8790-24b50ffe4409")) {                    Plotly.newPlot(                        "e8c48c58-6618-4eb6-8790-24b50ffe4409",                        [],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"value"},"type":"log"},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"index"}},"legend":{"tracegroupgap":0},"title":{"text":"Induction Loss over training: solu-2l"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="1dda3792-cb5b-475a-95ed-9a9f74c8cb4a" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("1dda3792-cb5b-475a-95ed-9a9f74c8cb4a")) {                    Plotly.newPlot(                        "1dda3792-cb5b-475a-95ed-9a9f74c8cb4a",                        [],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"value"},"type":"log"},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"index"}},"legend":{"tracegroupgap":0},"title":{"text":"Induction Loss over training: solu-2l"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('e8c48c58-6618-4eb6-8790-24b50ffe4409');
+var gd = document.getElementById('1dda3792-cb5b-475a-95ed-9a9f74c8cb4a');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2617,7 +2602,7 @@ <h3>Example: Induction Head Phase Transition<a class="headerlink" href="#Example
 </html></div>
 </div>
 <script type="application/vnd.jupyter.widget-state+json">
-{"state": {"2974465a897b4ac9ba7d573f8e1ea5f8": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "58a2ce5740e543bdaa92d598b1b084dc": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "977babee536f42158a1d81b4b7aa208d": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_2974465a897b4ac9ba7d573f8e1ea5f8", "max": 12.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_58a2ce5740e543bdaa92d598b1b084dc", "tabbable": null, "tooltip": null, "value": 12.0}}, "e4da68381e90404db807b341d42c96a8": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "4f9bedf73f97497db82c1b58ff5117bf": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "8480a1acfe1a4929becf5c1eeb2df5f1": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_e4da68381e90404db807b341d42c96a8", "placeholder": "\u200b", "style": "IPY_MODEL_4f9bedf73f97497db82c1b58ff5117bf", "tabbable": null, "tooltip": null, "value": "100%"}}, "22fa2830a0ab4ae4bb190db8ab08f59c": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "bd751bbba24c4154acc75b88e0f3a87a": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "fdf5d6542de342d08a2d9a0482eda1d7": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_22fa2830a0ab4ae4bb190db8ab08f59c", "placeholder": "\u200b", "style": "IPY_MODEL_bd751bbba24c4154acc75b88e0f3a87a", "tabbable": null, "tooltip": null, "value": "\u200712/12\u2007[00:19&lt;00:00,\u2007\u20071.65s/it]"}}, "76995f1052794da19381d92e27c1de96": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "ddd91d21a45e49a8a90fd6693fc0fe90": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_8480a1acfe1a4929becf5c1eeb2df5f1", "IPY_MODEL_977babee536f42158a1d81b4b7aa208d", "IPY_MODEL_fdf5d6542de342d08a2d9a0482eda1d7"], "layout": "IPY_MODEL_76995f1052794da19381d92e27c1de96", "tabbable": null, "tooltip": null}}, "ce2ad39062644901b02faaefcf98b910": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "16df306b166340878aae8ee0d7d2af94": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "4116100d429b4b9faf99f919b8cba2fd": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_ce2ad39062644901b02faaefcf98b910", "max": 352824413.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_16df306b166340878aae8ee0d7d2af94", "tabbable": null, "tooltip": null, "value": 352824413.0}}, "7f7ad53041c6499e9178b752e3f0bf66": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "a3feec14402a46afa537597c59f70647": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "c70847ad8e5543748ffbab29446b51c1": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_7f7ad53041c6499e9178b752e3f0bf66", "placeholder": "\u200b", "style": "IPY_MODEL_a3feec14402a46afa537597c59f70647", "tabbable": null, "tooltip": null, "value": "model.safetensors:\u2007100%"}}, "aa07cd1a18be4feda9dfaae9577809b5": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "5fbc42bb13f64054a6dbbd8c58ecfc59": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "6304f591778f49058f9403f77c821c56": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_aa07cd1a18be4feda9dfaae9577809b5", "placeholder": "\u200b", "style": "IPY_MODEL_5fbc42bb13f64054a6dbbd8c58ecfc59", "tabbable": null, "tooltip": null, "value": "\u2007353M/353M\u2007[00:01&lt;00:00,\u2007198MB/s]"}}, "1c150b40a42e403b848a74fd683a4788": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "2b653bed330e47bfaf1eb97604ffb148": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_c70847ad8e5543748ffbab29446b51c1", "IPY_MODEL_4116100d429b4b9faf99f919b8cba2fd", "IPY_MODEL_6304f591778f49058f9403f77c821c56"], "layout": "IPY_MODEL_1c150b40a42e403b848a74fd683a4788", "tabbable": null, "tooltip": null}}, "6e974e0f6e104aee841709766cbfe512": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "4025e90abd624e92ad601372fe03f72a": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "45071b25ff23496094a139022c8edc54": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_6e974e0f6e104aee841709766cbfe512", "max": 124.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_4025e90abd624e92ad601372fe03f72a", "tabbable": null, "tooltip": null, "value": 124.0}}, "19da29c1b3cf4fae9710cd70f0baf271": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "30e033f4a4a745f99416fa6c47e6b1b8": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "3a73a2ce3d684e57af5ead4d970bb546": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_19da29c1b3cf4fae9710cd70f0baf271", "placeholder": "\u200b", "style": "IPY_MODEL_30e033f4a4a745f99416fa6c47e6b1b8", "tabbable": null, "tooltip": null, "value": "generation_config.json:\u2007100%"}}, "3adaa7786f684ff6abb501929b91bfeb": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "236a3fe34dbb46b9a945a6c32dc4ed37": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "0093c35dd59c47adaed249b404d1a0b1": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_3adaa7786f684ff6abb501929b91bfeb", "placeholder": "\u200b", "style": "IPY_MODEL_236a3fe34dbb46b9a945a6c32dc4ed37", "tabbable": null, "tooltip": null, "value": "\u2007124/124\u2007[00:00&lt;00:00,\u200721.8kB/s]"}}, "2fbd1d8558174aaeae9046da37a2dc08": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "2f95bda34e124bdb8ad42b97968c2560": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_3a73a2ce3d684e57af5ead4d970bb546", "IPY_MODEL_45071b25ff23496094a139022c8edc54", "IPY_MODEL_0093c35dd59c47adaed249b404d1a0b1"], "layout": "IPY_MODEL_2fbd1d8558174aaeae9046da37a2dc08", "tabbable": null, "tooltip": null}}, "3383a9989d6a4375bccecc436f7df6c0": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "a05caff4cdcb42e8813ccaf3072ea092": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "f50d3b1ce11a44d8b5801bdd32a86fa8": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_3383a9989d6a4375bccecc436f7df6c0", "max": 26.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_a05caff4cdcb42e8813ccaf3072ea092", "tabbable": null, "tooltip": null, "value": 26.0}}, "f825f61e07764da39007e09a988d2df0": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "dee4a33c12ba4787ad99f56355b782af": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "d671580b566f41a3b09b9e31d0a97129": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_f825f61e07764da39007e09a988d2df0", "placeholder": "\u200b", "style": "IPY_MODEL_dee4a33c12ba4787ad99f56355b782af", "tabbable": null, "tooltip": null, "value": "tokenizer_config.json:\u2007100%"}}, "a95e7f64a7d240529cacf7a5a9e1c06b": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "40bb01d6e86d499a80c048f392bcbe29": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "a3a46ce06ec04834810c4ab96c959684": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_a95e7f64a7d240529cacf7a5a9e1c06b", "placeholder": "\u200b", "style": "IPY_MODEL_40bb01d6e86d499a80c048f392bcbe29", "tabbable": null, "tooltip": null, "value": "\u200726.0/26.0\u2007[00:00&lt;00:00,\u20075.03kB/s]"}}, "7129d439a9d549d48802b08586b8c24f": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "1bfe4dc8f9d64ed3a70a0e6e41b61d34": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_d671580b566f41a3b09b9e31d0a97129", "IPY_MODEL_f50d3b1ce11a44d8b5801bdd32a86fa8", "IPY_MODEL_a3a46ce06ec04834810c4ab96c959684"], "layout": "IPY_MODEL_7129d439a9d549d48802b08586b8c24f", "tabbable": null, "tooltip": null}}, "25f38d8b724d481397c21ffd4e2ae408": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "546e22db36d04372a0aa187adbcf0262": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "975c4858e139485f8f0aa84771e564a4": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_25f38d8b724d481397c21ffd4e2ae408", "max": 1042301.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_546e22db36d04372a0aa187adbcf0262", "tabbable": null, "tooltip": null, "value": 1042301.0}}, "bc420542d1d34e14978dcc6d9d0471e3": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "c741d615fd45485b899d67710062ed85": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "98db25d1775c4574b9702ae9d27f9d76": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_bc420542d1d34e14978dcc6d9d0471e3", "placeholder": "\u200b", "style": "IPY_MODEL_c741d615fd45485b899d67710062ed85", "tabbable": null, "tooltip": null, "value": "vocab.json:\u2007100%"}}, "355529cac414494c83e338e54c92a4cc": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "72e1c6506989403db938acb8a05caa9a": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "3b38cd8e7a4749a49d9442a541dd6d69": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_355529cac414494c83e338e54c92a4cc", "placeholder": "\u200b", "style": "IPY_MODEL_72e1c6506989403db938acb8a05caa9a", "tabbable": null, "tooltip": null, "value": "\u20071.04M/1.04M\u2007[00:00&lt;00:00,\u200717.5MB/s]"}}, "a6d329232ced49958c43f07f02862115": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "363a53f62bad40f38d9b57415fcdd27a": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_98db25d1775c4574b9702ae9d27f9d76", "IPY_MODEL_975c4858e139485f8f0aa84771e564a4", "IPY_MODEL_3b38cd8e7a4749a49d9442a541dd6d69"], "layout": "IPY_MODEL_a6d329232ced49958c43f07f02862115", "tabbable": null, "tooltip": null}}, "b49e96fd18bc45e981e58cefbd348841": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "4045d70e18ef41f892b496c2457200e8": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "4ec86ba49fc04f1ea410f2a37bf7ffae": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_b49e96fd18bc45e981e58cefbd348841", "max": 456318.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_4045d70e18ef41f892b496c2457200e8", "tabbable": null, "tooltip": null, "value": 456318.0}}, "46833033a8eb479ea4c5c87af7d997bb": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "5ad68d8e26e44e76a786ef4b57a2263d": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "aec5d70e86d84aef8fe50515973443a8": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_46833033a8eb479ea4c5c87af7d997bb", "placeholder": "\u200b", "style": "IPY_MODEL_5ad68d8e26e44e76a786ef4b57a2263d", "tabbable": null, "tooltip": null, "value": "merges.txt:\u2007100%"}}, "c6ee4432596b4ff999fe1bddd0d46240": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "697afa5733c6440f907ee2c1ffb2584e": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "4d16a5f6738f41918a9be1a5c6911d80": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_c6ee4432596b4ff999fe1bddd0d46240", "placeholder": "\u200b", "style": "IPY_MODEL_697afa5733c6440f907ee2c1ffb2584e", "tabbable": null, "tooltip": null, "value": "\u2007456k/456k\u2007[00:00&lt;00:00,\u200735.6MB/s]"}}, "485b5446c8864360a4f063c6992e772a": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "baa802cac1bf4a77bd2d6f2da4e0c206": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_aec5d70e86d84aef8fe50515973443a8", "IPY_MODEL_4ec86ba49fc04f1ea410f2a37bf7ffae", "IPY_MODEL_4d16a5f6738f41918a9be1a5c6911d80"], "layout": "IPY_MODEL_485b5446c8864360a4f063c6992e772a", "tabbable": null, "tooltip": null}}, "f5bf41cb59ed4db4b184f40d99bfbc1a": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "ab0021bee3b94de0965c25c38a7962f2": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "db49cd92e84f4f20b4d44fe5829790df": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_f5bf41cb59ed4db4b184f40d99bfbc1a", "max": 1355256.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_ab0021bee3b94de0965c25c38a7962f2", "tabbable": null, "tooltip": null, "value": 1355256.0}}, "9fce92e16f574800b2a9add4b6c10ef8": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "bfdbeaa99bc942abb6a71e94629483ab": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "0f1f32e4ccb44e77b4a1aab830f601a5": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_9fce92e16f574800b2a9add4b6c10ef8", "placeholder": "\u200b", "style": "IPY_MODEL_bfdbeaa99bc942abb6a71e94629483ab", "tabbable": null, "tooltip": null, "value": "tokenizer.json:\u2007100%"}}, "47b00b0a700a470cba9e0970646bf5a3": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "459712eff9964201b95e3cb31857e8da": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "70759237fca04d3084c2a5df2a30f271": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_47b00b0a700a470cba9e0970646bf5a3", "placeholder": "\u200b", "style": "IPY_MODEL_459712eff9964201b95e3cb31857e8da", "tabbable": null, "tooltip": null, "value": "\u20071.36M/1.36M\u2007[00:00&lt;00:00,\u20079.38MB/s]"}}, "c666ac6d4533412e922fc4b9a1084fc1": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "85239cb8d6ba4d288b001290e919d816": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_0f1f32e4ccb44e77b4a1aab830f601a5", "IPY_MODEL_db49cd92e84f4f20b4d44fe5829790df", "IPY_MODEL_70759237fca04d3084c2a5df2a30f271"], "layout": "IPY_MODEL_c666ac6d4533412e922fc4b9a1084fc1", "tabbable": null, "tooltip": null}}, "0347d987e2964dd4a31348434040c0c7": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "858356c58e784bc4b5778ed67747a0d1": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "69da672452a94ea68b438b3be68f1a24": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_0347d987e2964dd4a31348434040c0c7", "max": 50.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_858356c58e784bc4b5778ed67747a0d1", "tabbable": null, "tooltip": null, "value": 50.0}}, "33b3f2ed76434c13b68809069903b693": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "ae9c1384a8ef40a9b2a32844f1646716": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "1dfb46a04fe14273b07f0621a7a25096": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_33b3f2ed76434c13b68809069903b693", "placeholder": "\u200b", "style": "IPY_MODEL_ae9c1384a8ef40a9b2a32844f1646716", "tabbable": null, "tooltip": null, "value": "100%"}}, "e7fe152978d2459c9cf82d8f9f257479": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "a6544264d0d741aa81b06d6cf1050e84": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "38bdea7d9b64452ea8420690243965cf": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_e7fe152978d2459c9cf82d8f9f257479", "placeholder": "\u200b", "style": "IPY_MODEL_a6544264d0d741aa81b06d6cf1050e84", "tabbable": null, "tooltip": null, "value": "\u200750/50\u2007[00:02&lt;00:00,\u200722.25it/s]"}}, "217459b8733641fda6a21621dc147583": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "ef67630185584553867488aa7abe9fa9": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_1dfb46a04fe14273b07f0621a7a25096", "IPY_MODEL_69da672452a94ea68b438b3be68f1a24", "IPY_MODEL_38bdea7d9b64452ea8420690243965cf"], "layout": "IPY_MODEL_217459b8733641fda6a21621dc147583", "tabbable": null, "tooltip": null}}}, "version_major": 2, "version_minor": 0}
+{"state": {"fa2989d4907f487fb94956f67090d1bd": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "8c32daa5d0a64fcdaadad700b50b2e27": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "4f5a31f406bc4c8582c7878892bede4f": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_fa2989d4907f487fb94956f67090d1bd", "max": 12.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_8c32daa5d0a64fcdaadad700b50b2e27", "tabbable": null, "tooltip": null, "value": 12.0}}, "07913cfb30d7470ea6fe716d3599cbfd": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "d7e88b54c04d415fac710f14830d32b3": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "1e93a9f0f574448abab30220598d14ea": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_07913cfb30d7470ea6fe716d3599cbfd", "placeholder": "\u200b", "style": "IPY_MODEL_d7e88b54c04d415fac710f14830d32b3", "tabbable": null, "tooltip": null, "value": "100%"}}, "f650970bbec24731978e61f5639f83d8": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "359e2768fce646a0a615612fee3e508e": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "046865f8c6fa4333a2aafa4067066f4e": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_f650970bbec24731978e61f5639f83d8", "placeholder": "\u200b", "style": "IPY_MODEL_359e2768fce646a0a615612fee3e508e", "tabbable": null, "tooltip": null, "value": "\u200712/12\u2007[00:19&lt;00:00,\u2007\u20071.58s/it]"}}, "4d7242e8d4ec4e1db36416d52dedc096": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "bb02e41e51664a96b3c06abd21683006": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_1e93a9f0f574448abab30220598d14ea", "IPY_MODEL_4f5a31f406bc4c8582c7878892bede4f", "IPY_MODEL_046865f8c6fa4333a2aafa4067066f4e"], "layout": "IPY_MODEL_4d7242e8d4ec4e1db36416d52dedc096", "tabbable": null, "tooltip": null}}, "f8b7bd5b73384893a9643f6f0f3bc7a5": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "00a1e307bf234324ab8c80e2b11eeeee": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "214306a3558d400bb2e91903307496dc": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_f8b7bd5b73384893a9643f6f0f3bc7a5", "max": 352824413.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_00a1e307bf234324ab8c80e2b11eeeee", "tabbable": null, "tooltip": null, "value": 352824413.0}}, "18993102a6b540c6b4b3a8e65a3285a5": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "c420d4cb979d4cb6a77edaf29e09f965": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "26bec98fe7ff4d0f8c9f2668af368199": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_18993102a6b540c6b4b3a8e65a3285a5", "placeholder": "\u200b", "style": "IPY_MODEL_c420d4cb979d4cb6a77edaf29e09f965", "tabbable": null, "tooltip": null, "value": "model.safetensors:\u2007100%"}}, "411ab9797b3a44b4b5751b6df3a35e03": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "bd64574c041b44d38017dfd4820b2e4b": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "57e1eddf9db444169ab9759f877f7de7": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_411ab9797b3a44b4b5751b6df3a35e03", "placeholder": "\u200b", "style": "IPY_MODEL_bd64574c041b44d38017dfd4820b2e4b", "tabbable": null, "tooltip": null, "value": "\u2007353M/353M\u2007[00:01&lt;00:00,\u2007215MB/s]"}}, "95093d79632c42e89f65648356e52288": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "16f9e028019c47c19107395141087d9c": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_26bec98fe7ff4d0f8c9f2668af368199", "IPY_MODEL_214306a3558d400bb2e91903307496dc", "IPY_MODEL_57e1eddf9db444169ab9759f877f7de7"], "layout": "IPY_MODEL_95093d79632c42e89f65648356e52288", "tabbable": null, "tooltip": null}}, "e4a3d32f101647e0b844ff2ee427907a": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "18bf6e516c5442bfb281ac041eb68673": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "7c5cfa385cd8435ca3d7703e8106d131": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_e4a3d32f101647e0b844ff2ee427907a", "max": 124.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_18bf6e516c5442bfb281ac041eb68673", "tabbable": null, "tooltip": null, "value": 124.0}}, "01f4e058fc8a4ad68133f91434605cb6": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "a099ae8a89304ee0b2e630e61700dd39": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "33f927be3355434587a90f29e15b30a1": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_01f4e058fc8a4ad68133f91434605cb6", "placeholder": "\u200b", "style": "IPY_MODEL_a099ae8a89304ee0b2e630e61700dd39", "tabbable": null, "tooltip": null, "value": "generation_config.json:\u2007100%"}}, "82af6bbb20854ff98eedf4e5944fad63": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "b7f610cc1c1d418594f22fd76ac0e072": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "e648af8d85b046298af44b614c30d687": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_82af6bbb20854ff98eedf4e5944fad63", "placeholder": "\u200b", "style": "IPY_MODEL_b7f610cc1c1d418594f22fd76ac0e072", "tabbable": null, "tooltip": null, "value": "\u2007124/124\u2007[00:00&lt;00:00,\u200724.2kB/s]"}}, "877233dda9cb4d00864bf6470ae98ceb": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "8c41c6eefd75443fa69c9327b4c1ec8e": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_33f927be3355434587a90f29e15b30a1", "IPY_MODEL_7c5cfa385cd8435ca3d7703e8106d131", "IPY_MODEL_e648af8d85b046298af44b614c30d687"], "layout": "IPY_MODEL_877233dda9cb4d00864bf6470ae98ceb", "tabbable": null, "tooltip": null}}, "bec7b50387f340908a1ce664d0ee7e46": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "762ae9d728004d92a034a51cea8fec66": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "2c3822722a26406f9a3cfce27fa2f9e2": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_bec7b50387f340908a1ce664d0ee7e46", "max": 26.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_762ae9d728004d92a034a51cea8fec66", "tabbable": null, "tooltip": null, "value": 26.0}}, "ecb2ff817ce746eb85ba29b7cf393758": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "1bcf54ed3c7c4a20937255e48eb68170": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "704a2185ba1e43c18f631342bab60f7f": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_ecb2ff817ce746eb85ba29b7cf393758", "placeholder": "\u200b", "style": "IPY_MODEL_1bcf54ed3c7c4a20937255e48eb68170", "tabbable": null, "tooltip": null, "value": "tokenizer_config.json:\u2007100%"}}, "6f40fa1bcbbf4455817daaefafb9f5b6": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "4289a5d73d3241199ee38b1ef4ac475d": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "c42f7e12651d47c899dc8ae3cd954dd1": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_6f40fa1bcbbf4455817daaefafb9f5b6", "placeholder": "\u200b", "style": "IPY_MODEL_4289a5d73d3241199ee38b1ef4ac475d", "tabbable": null, "tooltip": null, "value": "\u200726.0/26.0\u2007[00:00&lt;00:00,\u20074.36kB/s]"}}, "391ab10e327244fa91d8ab82410c81ac": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "6a66981f155d4bf280913b398c75f705": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_704a2185ba1e43c18f631342bab60f7f", "IPY_MODEL_2c3822722a26406f9a3cfce27fa2f9e2", "IPY_MODEL_c42f7e12651d47c899dc8ae3cd954dd1"], "layout": "IPY_MODEL_391ab10e327244fa91d8ab82410c81ac", "tabbable": null, "tooltip": null}}, "77d883cf087b45fc9a7fc9786dadbe63": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "b995e0b517794c4a935c99155b7bd3c7": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "b1c430d051414eecaccd61435c0ed34c": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_77d883cf087b45fc9a7fc9786dadbe63", "max": 1042301.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_b995e0b517794c4a935c99155b7bd3c7", "tabbable": null, "tooltip": null, "value": 1042301.0}}, "b696d0501fbf4e0f8ca8123a5d9359d8": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "8dafe502a1d947ed89517420f0e8d1bd": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "dc7c542bbf5d4b6ca58dcec68b91e68c": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_b696d0501fbf4e0f8ca8123a5d9359d8", "placeholder": "\u200b", "style": "IPY_MODEL_8dafe502a1d947ed89517420f0e8d1bd", "tabbable": null, "tooltip": null, "value": "vocab.json:\u2007100%"}}, "699d9373f7ba45d490db037be91b05c5": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "f5ed129096c34e70bf382ba956e836bf": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "af82e06c82bf452fa1a9d0ff9dd9f131": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_699d9373f7ba45d490db037be91b05c5", "placeholder": "\u200b", "style": "IPY_MODEL_f5ed129096c34e70bf382ba956e836bf", "tabbable": null, "tooltip": null, "value": "\u20071.04M/1.04M\u2007[00:00&lt;00:00,\u20075.17MB/s]"}}, "1f0204d5124744658fb76eb12e63d0a4": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "59a6916e36b44608987b585683320e55": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_dc7c542bbf5d4b6ca58dcec68b91e68c", "IPY_MODEL_b1c430d051414eecaccd61435c0ed34c", "IPY_MODEL_af82e06c82bf452fa1a9d0ff9dd9f131"], "layout": "IPY_MODEL_1f0204d5124744658fb76eb12e63d0a4", "tabbable": null, "tooltip": null}}, "d333a600da514d26af421e9c420ff241": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "3bbe6453f4b7453ca7567b066697a38f": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "419c4221831f458d9160b21cdf5adb9d": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_d333a600da514d26af421e9c420ff241", "max": 456318.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_3bbe6453f4b7453ca7567b066697a38f", "tabbable": null, "tooltip": null, "value": 456318.0}}, "917ca83d84b043db937ff9901f09976a": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "bbcadaf9fda64c49bb69a0f9ee183beb": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "8a3b44f2c5d84e94a2bb4637a12748b9": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_917ca83d84b043db937ff9901f09976a", "placeholder": "\u200b", "style": "IPY_MODEL_bbcadaf9fda64c49bb69a0f9ee183beb", "tabbable": null, "tooltip": null, "value": "merges.txt:\u2007100%"}}, "82787d08d40948da9d57fa80681d6c01": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "895aa0507aab41399395c24c2da2a903": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "ffd00559da434d5ea3a744e89e8ab013": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_82787d08d40948da9d57fa80681d6c01", "placeholder": "\u200b", "style": "IPY_MODEL_895aa0507aab41399395c24c2da2a903", "tabbable": null, "tooltip": null, "value": "\u2007456k/456k\u2007[00:00&lt;00:00,\u20073.49MB/s]"}}, "03d7575b9cbc48da96ff17baa48be4c4": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "f462bf0c8aef4d9e9fd54629b8fe4434": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_8a3b44f2c5d84e94a2bb4637a12748b9", "IPY_MODEL_419c4221831f458d9160b21cdf5adb9d", "IPY_MODEL_ffd00559da434d5ea3a744e89e8ab013"], "layout": "IPY_MODEL_03d7575b9cbc48da96ff17baa48be4c4", "tabbable": null, "tooltip": null}}, "9b98600e30e241658a16044231674040": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "e123c944e91c4d53a6040ffeab016e69": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "c46d177181494ee1a7dd6e5c9c8c86fa": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_9b98600e30e241658a16044231674040", "max": 1355256.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_e123c944e91c4d53a6040ffeab016e69", "tabbable": null, "tooltip": null, "value": 1355256.0}}, "a3a591fc7c61495d86702fe435581acf": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "9b2871c6328f44629e38d82e383140ac": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "53e1b0c15a7b45b88c0377ead2d99333": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_a3a591fc7c61495d86702fe435581acf", "placeholder": "\u200b", "style": "IPY_MODEL_9b2871c6328f44629e38d82e383140ac", "tabbable": null, "tooltip": null, "value": "tokenizer.json:\u2007100%"}}, "cf524e5a513e435cab49c816b198a542": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "9a813b30c933431685c2e34827469d49": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "e9b4b7ca03a34748a25a569fa3dacc27": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_cf524e5a513e435cab49c816b198a542", "placeholder": "\u200b", "style": "IPY_MODEL_9a813b30c933431685c2e34827469d49", "tabbable": null, "tooltip": null, "value": "\u20071.36M/1.36M\u2007[00:00&lt;00:00,\u20076.86MB/s]"}}, "faa2abf5d66c4dfe96ce5fec6486692e": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "29b6d4431334417e9690be62ac184ff3": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_53e1b0c15a7b45b88c0377ead2d99333", "IPY_MODEL_c46d177181494ee1a7dd6e5c9c8c86fa", "IPY_MODEL_e9b4b7ca03a34748a25a569fa3dacc27"], "layout": "IPY_MODEL_faa2abf5d66c4dfe96ce5fec6486692e", "tabbable": null, "tooltip": null}}, "949dd78c7d124e8ab4696ed732b1de5c": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "333364115678404d89398e4c1e998ef3": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "9491a41bb00f49c9ac6243e9b050e8ea": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_949dd78c7d124e8ab4696ed732b1de5c", "max": 50.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_333364115678404d89398e4c1e998ef3", "tabbable": null, "tooltip": null, "value": 50.0}}, "a92a6cd4defc4e12ba9b9ff94af36e62": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "dc3cea0e6cea4325879ccc3a197abc5f": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "f65616c888c944d3a7778e70fcccebf0": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_a92a6cd4defc4e12ba9b9ff94af36e62", "placeholder": "\u200b", "style": "IPY_MODEL_dc3cea0e6cea4325879ccc3a197abc5f", "tabbable": null, "tooltip": null, "value": "100%"}}, "85c96ee49b1e46fdafef7b1f934ef9ad": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "30ad307a0269480eb94b844695317e07": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "fca8d2f8306640ba9f2e11c6d492392e": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_85c96ee49b1e46fdafef7b1f934ef9ad", "placeholder": "\u200b", "style": "IPY_MODEL_30ad307a0269480eb94b844695317e07", "tabbable": null, "tooltip": null, "value": "\u200750/50\u2007[00:02&lt;00:00,\u200723.96it/s]"}}, "11156915bf414366b67a6a0f9e6c1960": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "428a330dae074773aad72f7de9f6b219": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_f65616c888c944d3a7778e70fcccebf0", "IPY_MODEL_9491a41bb00f49c9ac6243e9b050e8ea", "IPY_MODEL_fca8d2f8306640ba9f2e11c6d492392e"], "layout": "IPY_MODEL_11156915bf414366b67a6a0f9e6c1960", "tabbable": null, "tooltip": null}}}, "version_major": 2, "version_minor": 0}
 </script></section>
 </section>
 </section>
diff --git a/generated/demos/Main_Demo.ipynb b/generated/demos/Main_Demo.ipynb
index ac34c5192..11da2649d 100644
--- a/generated/demos/Main_Demo.ipynb
+++ b/generated/demos/Main_Demo.ipynb
@@ -48,10 +48,10 @@
    "execution_count": 1,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:21.187448Z",
-     "iopub.status.busy": "2024-12-14T01:01:21.187251Z",
-     "iopub.status.idle": "2024-12-14T01:01:21.197652Z",
-     "shell.execute_reply": "2024-12-14T01:01:21.197226Z"
+     "iopub.execute_input": "2024-12-31T02:20:38.485505Z",
+     "iopub.status.busy": "2024-12-31T02:20:38.485333Z",
+     "iopub.status.idle": "2024-12-31T02:20:38.495439Z",
+     "shell.execute_reply": "2024-12-31T02:20:38.495010Z"
     }
    },
    "outputs": [],
@@ -89,10 +89,10 @@
    "execution_count": 2,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:21.199828Z",
-     "iopub.status.busy": "2024-12-14T01:01:21.199477Z",
-     "iopub.status.idle": "2024-12-14T01:01:21.320090Z",
-     "shell.execute_reply": "2024-12-14T01:01:21.319483Z"
+     "iopub.execute_input": "2024-12-31T02:20:38.497449Z",
+     "iopub.status.busy": "2024-12-31T02:20:38.497279Z",
+     "iopub.status.idle": "2024-12-31T02:20:38.618494Z",
+     "shell.execute_reply": "2024-12-31T02:20:38.617983Z"
     }
    },
    "outputs": [
@@ -119,28 +119,28 @@
    "execution_count": 3,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:21.322347Z",
-     "iopub.status.busy": "2024-12-14T01:01:21.321942Z",
-     "iopub.status.idle": "2024-12-14T01:01:22.768775Z",
-     "shell.execute_reply": "2024-12-14T01:01:22.768119Z"
+     "iopub.execute_input": "2024-12-31T02:20:38.620634Z",
+     "iopub.status.busy": "2024-12-31T02:20:38.620292Z",
+     "iopub.status.idle": "2024-12-31T02:20:39.997450Z",
+     "shell.execute_reply": "2024-12-31T02:20:39.996806Z"
     }
    },
    "outputs": [
     {
      "data": {
       "text/html": [
-       "<div id=\"circuits-vis-48c8c518-76ab\" style=\"margin: 15px 0;\"/>\n",
+       "<div id=\"circuits-vis-88c92a02-5c72\" style=\"margin: 15px 0;\"/>\n",
        "    <script crossorigin type=\"module\">\n",
        "    import { render, Hello } from \"https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js\";\n",
        "    render(\n",
-       "      \"circuits-vis-48c8c518-76ab\",\n",
+       "      \"circuits-vis-88c92a02-5c72\",\n",
        "      Hello,\n",
        "      {\"name\": \"Neel\"}\n",
        "    )\n",
        "    </script>"
       ],
       "text/plain": [
-       "<circuitsvis.utils.render.RenderedHTML at 0x7f0f58854a90>"
+       "<circuitsvis.utils.render.RenderedHTML at 0x7f028e0f8b50>"
       ]
      },
      "execution_count": 3,
@@ -159,10 +159,10 @@
    "execution_count": 4,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:22.771137Z",
-     "iopub.status.busy": "2024-12-14T01:01:22.770703Z",
-     "iopub.status.idle": "2024-12-14T01:01:23.078839Z",
-     "shell.execute_reply": "2024-12-14T01:01:23.078261Z"
+     "iopub.execute_input": "2024-12-31T02:20:39.999792Z",
+     "iopub.status.busy": "2024-12-31T02:20:39.999253Z",
+     "iopub.status.idle": "2024-12-31T02:20:40.296914Z",
+     "shell.execute_reply": "2024-12-31T02:20:40.296353Z"
     }
    },
    "outputs": [],
@@ -184,10 +184,10 @@
    "execution_count": 5,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:23.081471Z",
-     "iopub.status.busy": "2024-12-14T01:01:23.081005Z",
-     "iopub.status.idle": "2024-12-14T01:01:24.683057Z",
-     "shell.execute_reply": "2024-12-14T01:01:24.682500Z"
+     "iopub.execute_input": "2024-12-31T02:20:40.299549Z",
+     "iopub.status.busy": "2024-12-31T02:20:40.299071Z",
+     "iopub.status.idle": "2024-12-31T02:20:41.792633Z",
+     "shell.execute_reply": "2024-12-31T02:20:41.792089Z"
     }
    },
    "outputs": [],
@@ -212,17 +212,17 @@
    "execution_count": 6,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:24.685764Z",
-     "iopub.status.busy": "2024-12-14T01:01:24.685314Z",
-     "iopub.status.idle": "2024-12-14T01:01:24.689311Z",
-     "shell.execute_reply": "2024-12-14T01:01:24.688764Z"
+     "iopub.execute_input": "2024-12-31T02:20:41.795158Z",
+     "iopub.status.busy": "2024-12-31T02:20:41.794819Z",
+     "iopub.status.idle": "2024-12-31T02:20:41.798506Z",
+     "shell.execute_reply": "2024-12-31T02:20:41.798014Z"
     }
    },
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "<torch.autograd.grad_mode.set_grad_enabled at 0x7f0f50f9df90>"
+       "<torch.autograd.grad_mode.set_grad_enabled at 0x7f027eb64110>"
       ]
      },
      "execution_count": 6,
@@ -246,10 +246,10 @@
    "execution_count": 7,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:24.691560Z",
-     "iopub.status.busy": "2024-12-14T01:01:24.691109Z",
-     "iopub.status.idle": "2024-12-14T01:01:24.695875Z",
-     "shell.execute_reply": "2024-12-14T01:01:24.695298Z"
+     "iopub.execute_input": "2024-12-31T02:20:41.800537Z",
+     "iopub.status.busy": "2024-12-31T02:20:41.800217Z",
+     "iopub.status.idle": "2024-12-31T02:20:41.805545Z",
+     "shell.execute_reply": "2024-12-31T02:20:41.805108Z"
     }
    },
    "outputs": [],
@@ -301,10 +301,10 @@
    "execution_count": 8,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:24.697925Z",
-     "iopub.status.busy": "2024-12-14T01:01:24.697623Z",
-     "iopub.status.idle": "2024-12-14T01:01:24.700565Z",
-     "shell.execute_reply": "2024-12-14T01:01:24.700031Z"
+     "iopub.execute_input": "2024-12-31T02:20:41.807533Z",
+     "iopub.status.busy": "2024-12-31T02:20:41.807198Z",
+     "iopub.status.idle": "2024-12-31T02:20:41.810084Z",
+     "shell.execute_reply": "2024-12-31T02:20:41.809518Z"
     }
    },
    "outputs": [],
@@ -317,10 +317,10 @@
    "execution_count": 9,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:24.702501Z",
-     "iopub.status.busy": "2024-12-14T01:01:24.702210Z",
-     "iopub.status.idle": "2024-12-14T01:01:25.742888Z",
-     "shell.execute_reply": "2024-12-14T01:01:25.742226Z"
+     "iopub.execute_input": "2024-12-31T02:20:41.812094Z",
+     "iopub.status.busy": "2024-12-31T02:20:41.811693Z",
+     "iopub.status.idle": "2024-12-31T02:20:42.998550Z",
+     "shell.execute_reply": "2024-12-31T02:20:42.997926Z"
     }
    },
    "outputs": [
@@ -354,10 +354,10 @@
    "execution_count": 10,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:25.745568Z",
-     "iopub.status.busy": "2024-12-14T01:01:25.745062Z",
-     "iopub.status.idle": "2024-12-14T01:01:26.063199Z",
-     "shell.execute_reply": "2024-12-14T01:01:26.062608Z"
+     "iopub.execute_input": "2024-12-31T02:20:43.001107Z",
+     "iopub.status.busy": "2024-12-31T02:20:43.000656Z",
+     "iopub.status.idle": "2024-12-31T02:20:43.306400Z",
+     "shell.execute_reply": "2024-12-31T02:20:43.305797Z"
     }
    },
    "outputs": [
@@ -399,10 +399,10 @@
    "execution_count": 11,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:26.065580Z",
-     "iopub.status.busy": "2024-12-14T01:01:26.065191Z",
-     "iopub.status.idle": "2024-12-14T01:01:26.204404Z",
-     "shell.execute_reply": "2024-12-14T01:01:26.203778Z"
+     "iopub.execute_input": "2024-12-31T02:20:43.308834Z",
+     "iopub.status.busy": "2024-12-31T02:20:43.308482Z",
+     "iopub.status.idle": "2024-12-31T02:20:43.558974Z",
+     "shell.execute_reply": "2024-12-31T02:20:43.558327Z"
     }
    },
    "outputs": [
@@ -440,10 +440,10 @@
    "execution_count": 12,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:26.206966Z",
-     "iopub.status.busy": "2024-12-14T01:01:26.206475Z",
-     "iopub.status.idle": "2024-12-14T01:01:26.210963Z",
-     "shell.execute_reply": "2024-12-14T01:01:26.210505Z"
+     "iopub.execute_input": "2024-12-31T02:20:43.561897Z",
+     "iopub.status.busy": "2024-12-31T02:20:43.561436Z",
+     "iopub.status.idle": "2024-12-31T02:20:43.566067Z",
+     "shell.execute_reply": "2024-12-31T02:20:43.565503Z"
     }
    },
    "outputs": [
@@ -468,10 +468,10 @@
    "execution_count": 13,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:26.212839Z",
-     "iopub.status.busy": "2024-12-14T01:01:26.212666Z",
-     "iopub.status.idle": "2024-12-14T01:01:26.324357Z",
-     "shell.execute_reply": "2024-12-14T01:01:26.323744Z"
+     "iopub.execute_input": "2024-12-31T02:20:43.568146Z",
+     "iopub.status.busy": "2024-12-31T02:20:43.567836Z",
+     "iopub.status.idle": "2024-12-31T02:20:43.676139Z",
+     "shell.execute_reply": "2024-12-31T02:20:43.675604Z"
     }
    },
    "outputs": [
@@ -485,18 +485,18 @@
     {
      "data": {
       "text/html": [
-       "<div id=\"circuits-vis-ac0a9437-0fc4\" style=\"margin: 15px 0;\"/>\n",
+       "<div id=\"circuits-vis-90f99dc5-665a\" style=\"margin: 15px 0;\"/>\n",
        "    <script crossorigin type=\"module\">\n",
        "    import { render, AttentionPatterns } from \"https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js\";\n",
        "    render(\n",
-       "      \"circuits-vis-ac0a9437-0fc4\",\n",
+       "      \"circuits-vis-90f99dc5-665a\",\n",
        "      AttentionPatterns,\n",
        "      {\"tokens\": [\"<|endoftext|>\", \"Natural\", \" language\", \" processing\", \" tasks\", \",\", \" such\", \" as\", \" question\", \" answering\", \",\", \" machine\", \" translation\", \",\", \" reading\", \" comprehension\", \",\", \" and\", \" summar\", \"ization\", \",\", \" are\", \" typically\", \" approached\", \" with\", \" supervised\", \" learning\", \" on\", \" tasks\", \"pe\", \"cific\", \" datasets\", \".\"], \"attention\": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9639418125152588, 0.03605816140770912, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8389372825622559, 0.11828788369894028, 0.04277484491467476, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.47436124086380005, 0.13382026553153992, 0.27371731400489807, 0.11810113489627838, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.35606440901756287, 0.10184910148382187, 0.23054221272468567, 0.20397402346134186, 0.10757026076316833, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6660143733024597, 0.16866375505924225, 0.04535674676299095, 0.038855016231536865, 0.0677548199892044, 0.013355279341340065, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.38626962900161743, 0.285109281539917, 0.07609007507562637, 0.05908379331231117, 0.07223352044820786, 0.03979635238647461, 0.08141742646694183, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3775395154953003, 0.1883881539106369, 0.11723985522985458, 0.08685600012540817, 0.0666918158531189, 0.03500017523765564, 0.09693004935979843, 0.03135441616177559, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4869752824306488, 0.06781317293643951, 0.07952874153852463, 0.0848078578710556, 0.1590261608362198, 0.029577823355793953, 0.025685923174023628, 0.016474608331918716, 0.05011039599776268, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2906550168991089, 0.0401349738240242, 0.14614859223365784, 0.09940590709447861, 0.1538919061422348, 0.039001598954200745, 0.024988971650600433, 0.03184127062559128, 0.10222824662923813, 0.0717034563422203, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.39624103903770447, 0.09694176912307739, 0.027270659804344177, 0.02355135791003704, 0.03723449259996414, 0.006502415519207716, 0.08118756860494614, 0.013088458217680454, 0.06990590691566467, 0.24043099582195282, 0.007645336911082268, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.24864791333675385, 0.1380205899477005, 0.0923532024025917, 0.08676131069660187, 0.1381969302892685, 0.05914197862148285, 0.03223859518766403, 0.03158237785100937, 0.030489422380924225, 0.03873484209179878, 0.06671839207410812, 0.037114467471838, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1914844512939453, 0.1617259681224823, 0.07445938140153885, 0.07740949839353561, 0.021961113438010216, 0.03392130509018898, 0.05125021934509277, 0.01951923966407776, 0.03132446110248566, 0.04020152986049652, 0.038742680102586746, 0.21578852832317352, 0.042211681604385376, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3704318404197693, 0.08681417256593704, 0.02458467148244381, 0.021616315469145775, 0.032388731837272644, 0.005422737915068865, 0.0727522075176239, 0.0112727927044034, 0.06329693645238876, 0.21726809442043304, 0.006367155350744724, 0.029603807255625725, 0.05099846422672272, 0.007182050962001085, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1973765343427658, 0.04603995382785797, 0.0439998134970665, 0.1337345838546753, 0.05424821376800537, 0.025475721806287766, 0.027563506737351418, 0.021570932120084763, 0.051718227565288544, 0.06458097696304321, 0.02806464210152626, 0.23551592230796814, 0.019129814580082893, 0.029963519424200058, 0.021017681807279587, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08907236158847809, 0.01928834617137909, 0.1665353924036026, 0.07281264662742615, 0.04738640412688255, 0.024487905204296112, 0.028987322002649307, 0.019370367750525475, 0.026673022657632828, 0.07316635549068451, 0.025704585015773773, 0.04242359474301338, 0.058694612234830856, 0.028932694345712662, 0.18119072914123535, 0.09527372568845749, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2816729247570038, 0.06441289931535721, 0.018008552491664886, 0.01616962067782879, 0.023183872923254967, 0.0037532988935709, 0.05472247675061226, 0.007909760810434818, 0.046164702624082565, 0.16947267949581146, 0.004361647181212902, 0.021011337637901306, 0.0354907400906086, 0.004932564217597246, 0.0955522358417511, 0.14726325869560242, 0.005917454604059458, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.21305488049983978, 0.0591236874461174, 0.03382088243961334, 0.02747686579823494, 0.02839355543255806, 0.008422904647886753, 0.0400853231549263, 0.011629256419837475, 0.05295189097523689, 0.15404635667800903, 0.009831804782152176, 0.03610190004110336, 0.04737289249897003, 0.011069180443882942, 0.09972473233938217, 0.1397135704755783, 0.013185348361730576, 0.013994931243360043, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15871697664260864, 0.04387888312339783, 0.08712151646614075, 0.08998466283082962, 0.030738582834601402, 0.0341489352285862, 0.024917256087064743, 0.03139195218682289, 0.024823859333992004, 0.019790329039096832, 0.03625484183430672, 0.020694412291049957, 0.042840685695409775, 0.03820899501442909, 0.06234658882021904, 0.10919700562953949, 0.0413760244846344, 0.04916759952902794, 0.054400913417339325, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10485509783029556, 0.12122292071580887, 0.0648748055100441, 0.08768714964389801, 0.03434053063392639, 0.01748395338654518, 0.034151818603277206, 0.015289156697690487, 0.023312119767069817, 0.028306512162089348, 0.01872047781944275, 0.028111934661865234, 0.041905295103788376, 0.0209895521402359, 0.04678506404161453, 0.08659638464450836, 0.023631852120161057, 0.024273162707686424, 0.16702400147914886, 0.010438223369419575, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.22901973128318787, 0.05184381827712059, 0.013585173524916172, 0.012337238527834415, 0.018005413934588432, 0.0027703619562089443, 0.042381297796964645, 0.005856257397681475, 0.03614485636353493, 0.13039222359657288, 0.0031534270383417606, 0.01567256823182106, 0.027800394222140312, 0.003554322523996234, 0.07460816204547882, 0.11298283189535141, 0.004272268619388342, 0.006832208018749952, 0.18569746613502502, 0.018073637038469315, 0.005016352981328964, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.18869927525520325, 0.03438714146614075, 0.022344734519720078, 0.019972724840044975, 0.016354026272892952, 0.0068565551191568375, 0.020859038457274437, 0.005695997271686792, 0.03415916487574577, 0.07260986417531967, 0.007857211865484715, 0.018040239810943604, 0.026904456317424774, 0.009020394645631313, 0.06876447051763535, 0.17578734457492828, 0.010720067657530308, 0.00928453542292118, 0.1925639659166336, 0.025180332362651825, 0.012639074586331844, 0.021299341693520546, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1195831298828125, 0.022259412333369255, 0.032947149127721786, 0.020170224830508232, 0.035653311759233475, 0.013459913432598114, 0.017516475170850754, 0.010057873092591763, 0.025856440886855125, 0.05955953523516655, 0.015084508806467056, 0.015008730813860893, 0.053174685686826706, 0.016597608104348183, 0.041555255651474, 0.131293386220932, 0.01929667219519615, 0.015855036675930023, 0.1792508363723755, 0.01618383638560772, 0.02229553461074829, 0.015463392250239849, 0.10187702625989914, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14615531265735626, 0.026727521792054176, 0.01662455126643181, 0.018987685441970825, 0.06278638541698456, 0.015317163430154324, 0.019792240113019943, 0.01422776561230421, 0.025458168238401413, 0.045303549617528915, 0.016364356502890587, 0.037493038922548294, 0.0132886478677392, 0.017496541142463684, 0.0399458184838295, 0.05881758779287338, 0.01926097832620144, 0.024616025388240814, 0.038219697773456573, 0.02157779224216938, 0.02094990760087967, 0.07973217219114304, 0.05017608404159546, 0.17068105936050415, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11321667581796646, 0.03674636408686638, 0.011786575429141521, 0.010274871252477169, 0.020370664075016975, 0.00524388812482357, 0.015918847173452377, 0.005266785155981779, 0.02489173412322998, 0.06593258678913116, 0.005933663807809353, 0.018209032714366913, 0.021020200103521347, 0.0066674984991550446, 0.034828800708055496, 0.13742128014564514, 0.007927048951387405, 0.008618664927780628, 0.1137719452381134, 0.013557437807321548, 0.009277833625674248, 0.026121344417333603, 0.08499342203140259, 0.19073909521102905, 0.011263742111623287, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13337711989879608, 0.026216603815555573, 0.03827154263854027, 0.07152578234672546, 0.05317767709493637, 0.013925841078162193, 0.007084188051521778, 0.013450146652758121, 0.009841442108154297, 0.011789779178798199, 0.013537583872675896, 0.038154955953359604, 0.041933052241802216, 0.013882285915315151, 0.03707146272063255, 0.138384610414505, 0.014846334233880043, 0.03156952187418938, 0.05598173290491104, 0.015536684542894363, 0.01595635898411274, 0.045455560088157654, 0.01669965498149395, 0.02532576024532318, 0.03671892359852791, 0.08028541505336761, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10608974099159241, 0.019107727333903313, 0.024468660354614258, 0.027496395632624626, 0.016365809366106987, 0.0050114234909415245, 0.010413105599582195, 0.006081144325435162, 0.005301064345985651, 0.011143166571855545, 0.004565386101603508, 0.01896991953253746, 0.004321120213717222, 0.00481497822329402, 0.02940940298140049, 0.028682034462690353, 0.005097254645079374, 0.0072343479841947556, 0.03412593528628349, 0.010370595380663872, 0.005643266253173351, 0.007283586077392101, 0.029389560222625732, 0.010038800537586212, 0.009134513325989246, 0.546663224697113, 0.012777911499142647, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10467305034399033, 0.03321940079331398, 0.015341250225901604, 0.009373549371957779, 0.026595456525683403, 0.0057878089137375355, 0.013571349903941154, 0.004554884973913431, 0.028058893978595734, 0.026107225567102432, 0.006353434175252914, 0.013315831311047077, 0.026628265157341957, 0.006888872943818569, 0.06204749271273613, 0.05890703946352005, 0.008068050257861614, 0.007557098753750324, 0.0852278545498848, 0.017075754702091217, 0.009256886318325996, 0.019695758819580078, 0.12617813050746918, 0.13061513006687164, 0.011351036839187145, 0.08984372764825821, 0.04638150706887245, 0.007325289770960808, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08561903238296509, 0.021438946947455406, 0.056412599980831146, 0.0570666640996933, 0.019802208989858627, 0.00672749662771821, 0.005809164140373468, 0.004516261164098978, 0.0031647481955587864, 0.01761520653963089, 0.006174598820507526, 0.08767975866794586, 0.012299864552915096, 0.006350455805659294, 0.017522143200039864, 0.14295215904712677, 0.006585489492863417, 0.007875688374042511, 0.030078941956162453, 0.01390750240534544, 0.0073766945861279964, 0.007684784941375256, 0.02216084487736225, 0.012385032139718533, 0.011890063062310219, 0.08669772744178772, 0.19902527332305908, 0.01359447930008173, 0.029586246237158775, 0.0, 0.0, 0.0, 0.0], [0.14064587652683258, 0.013298707082867622, 0.015702949836850166, 0.017357859760522842, 0.02233150415122509, 0.029672270640730858, 0.04172082245349884, 0.01899542473256588, 0.03827710077166557, 0.048635613173246384, 0.03094690665602684, 0.016023898497223854, 0.020880894735455513, 0.032438233494758606, 0.030558118596673012, 0.022808339446783066, 0.03537759929895401, 0.031451594084501266, 0.03497113287448883, 0.01867910660803318, 0.03821907937526703, 0.02257886528968811, 0.06819561868906021, 0.04214095324277878, 0.028620855882763863, 0.03775003179907799, 0.018578052520751953, 0.03376871719956398, 0.03641696274280548, 0.012956855818629265, 0.0, 0.0, 0.0], [0.07168620079755783, 0.06924446672201157, 0.019306905567646027, 0.014161979779601097, 0.016823193058371544, 0.0193806029856205, 0.019257402047514915, 0.0220036581158638, 0.013706518337130547, 0.03578377887606621, 0.018465185537934303, 0.05207168683409691, 0.020085178315639496, 0.01986212469637394, 0.02066212333738804, 0.04725164547562599, 0.021076705306768417, 0.036787249147892, 0.024324089288711548, 0.0038275483530014753, 0.023920675739645958, 0.008533227257430553, 0.026241622865200043, 0.027380064129829407, 0.03461199253797531, 0.022884156554937363, 0.10047907382249832, 0.06913496553897858, 0.025474581867456436, 0.06495600938796997, 0.03061538189649582, 0.0, 0.0], [0.0691077709197998, 0.0370122492313385, 0.03862115740776062, 0.05933326855301857, 0.015923550352454185, 0.007918563671410084, 0.010371048003435135, 0.0066156634129583836, 0.002520075300708413, 0.026019366458058357, 0.007905230857431889, 0.02965201810002327, 0.04000623896718025, 0.008451344445347786, 0.010741154663264751, 0.05027563124895096, 0.009428859688341618, 0.013601033948361874, 0.05036921799182892, 0.03176714852452278, 0.010793027468025684, 0.007216801401227713, 0.006478427443653345, 0.01480061188340187, 0.021585967391729355, 0.15769484639167786, 0.08884759992361069, 0.0190169308334589, 0.02972934953868389, 0.03316137194633484, 0.050883661955595016, 0.034150850027799606, 0.0], [0.14375509321689606, 0.01681104488670826, 0.009386662393808365, 0.006830313708633184, 0.011656845919787884, 0.0015672279987484217, 0.019711481407284737, 0.002398042706772685, 0.021235886961221695, 0.04683680087327957, 0.0016905148513615131, 0.0058271917514503, 0.011979999952018261, 0.0018251396249979734, 0.042313333600759506, 0.054913729429244995, 0.0021786566358059645, 0.0024170097894966602, 0.09604065865278244, 0.005752880126237869, 0.0025577889755368233, 0.0071212234906852245, 0.08889931440353394, 0.10852081328630447, 0.005179054103791714, 0.03657734394073486, 0.02471994049847126, 0.0037347625475376844, 0.031077096238732338, 0.016887947916984558, 0.09450862556695938, 0.07171519845724106, 0.0033723204396665096]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0004246663593221456, 0.9995753169059753, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0005621908348985016, 0.016407281160354614, 0.9830306172370911, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0011627554195001721, 0.021681973710656166, 0.0037620372604578733, 0.9733933210372925, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [3.7244368286337703e-05, 0.00017202268645633012, 0.0002814392792060971, 0.0027421435806900263, 0.9967671632766724, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00826845783740282, 0.00023985625011846423, 7.361917960224673e-05, 6.43773382762447e-05, 0.0001756635756464675, 0.9911779761314392, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0012215041788294911, 0.005400444380939007, 0.0016716312384232879, 0.00040775578236207366, 0.0006163655198179185, 0.001093117636628449, 0.989589273929596, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0012459794525057077, 0.0009121228358708322, 0.0005976726533845067, 0.00013656896771863103, 0.00033041107235476375, 0.001572280889376998, 0.0038808276876807213, 0.9913240671157837, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00028217516955919564, 0.004068182315677404, 0.0026605194434523582, 0.0013093105517327785, 0.008030476048588753, 0.00028790938085876405, 0.00022922940843272954, 0.0003948427038267255, 0.9827372431755066, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [3.47392306139227e-05, 0.0003953832492697984, 0.00013272685464471579, 0.0002585221081972122, 0.001085555530153215, 9.198026964440942e-05, 0.000326707202475518, 0.000542744412086904, 0.006105918437242508, 0.9910256266593933, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0033785062842071056, 5.0908744015032426e-05, 1.6452069758088328e-05, 1.69261602422921e-05, 4.18141353293322e-05, 0.49394020438194275, 0.00012981283362023532, 0.0008837337954901159, 3.221205042791553e-05, 2.7252061045146547e-05, 0.5014821887016296, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [8.416055788984522e-05, 0.0013417234877124429, 0.001261359779164195, 0.0021450764033943415, 0.004042366519570351, 0.00048305385280400515, 0.0001158266604761593, 0.00015203595103230327, 2.6925330530502833e-05, 0.00012675137259066105, 0.00031289938488043845, 0.9899077415466309, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0003782230196520686, 0.0009837561519816518, 0.03934125974774361, 0.002732245484367013, 0.0036680190823972225, 0.00011039189848816022, 0.00012931032688356936, 0.00021743457182310522, 0.00010623293928802013, 0.0007748182397335768, 6.647665577474982e-05, 0.0003148667747154832, 0.9511768817901611, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0021017189137637615, 2.443686025799252e-05, 7.78878529672511e-06, 8.651618372823577e-06, 2.014001256611664e-05, 0.2997134327888489, 7.525274122599512e-05, 0.0004898307379335165, 1.845947736001108e-05, 1.534453986096196e-05, 0.32833898067474365, 4.175798676442355e-05, 6.469185791502241e-06, 0.3691376745700836, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0001496832846896723, 0.00011296618322376162, 0.00036294886376708746, 0.00018591222760733217, 0.00016460890765301883, 4.1432296711718664e-05, 2.8764745366061106e-05, 7.786958303768188e-05, 0.0009200992644764483, 0.0103401904925704, 2.7572339604375884e-05, 1.7833237507147714e-05, 0.0003305452992208302, 2.4375704015255906e-05, 0.9872152805328369, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00010753834794741124, 0.0021781916730105877, 0.002042611828073859, 0.004251930397003889, 0.006989913992583752, 2.511881393729709e-05, 0.0007779046427458525, 0.0005783551605418324, 0.00293784704990685, 0.033225417137145996, 1.71992760442663e-05, 0.0008936497615650296, 0.0015238587511703372, 1.4656818166258745e-05, 0.00622264388948679, 0.9382131695747375, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.001363488263450563, 1.3226085684436839e-05, 4.013950274384115e-06, 4.803066076419782e-06, 1.0257452231599018e-05, 0.19665955007076263, 4.527267810772173e-05, 0.0002776262117549777, 1.1714444553945214e-05, 9.473311365582049e-06, 0.22919613122940063, 2.6430649086250924e-05, 4.101847935089609e-06, 0.26576095819473267, 8.515355148119852e-06, 4.536029791779583e-06, 0.30659976601600647, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0008924751309677958, 0.00013490811397787184, 4.779841037816368e-05, 5.8037036069436e-05, 0.00010480164928594604, 0.012799090705811977, 0.0007168247830122709, 0.032579511404037476, 2.6449959477758966e-05, 0.00011185064795427024, 0.011884261853992939, 4.010266638942994e-05, 5.555404641199857e-05, 0.012377863749861717, 0.00010783471225295216, 5.4043663112679496e-05, 0.013122126460075378, 0.914886474609375, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [5.075309672974981e-06, 4.396129952510819e-05, 3.398501576157287e-05, 7.940301293274388e-05, 5.477911327034235e-05, 7.921535711830074e-07, 9.313323971582577e-06, 7.727071533736307e-06, 8.597254054620862e-05, 0.00012274031178094447, 5.141479277881444e-07, 1.702793611002562e-06, 3.83417536795605e-05, 4.4509667418424215e-07, 0.00013928221596870571, 0.0003275797644164413, 3.9947968843989656e-07, 3.948126504838001e-06, 0.999043881893158, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [9.346180740976706e-05, 0.0018395738443359733, 0.0025233125779777765, 0.018087238073349, 0.0029363748617470264, 0.00027335810591466725, 4.872974750469439e-05, 0.0004212774510961026, 0.00015624347724951804, 0.0009748333832249045, 0.00020533586211968213, 0.0010228825267404318, 0.0019548034761101007, 0.0001947036653291434, 0.0011294205905869603, 0.0016656103543937206, 0.00018734042532742023, 0.0009503448964096606, 0.0004455175076145679, 0.964889645576477, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0010859702015295625, 8.512333806720562e-06, 2.5307288069598144e-06, 3.0625205909018405e-06, 5.975129170110449e-06, 0.13292615115642548, 3.345326331327669e-05, 0.00018891248328145593, 8.477756637148559e-06, 6.540415142808342e-06, 0.1644592434167862, 1.813003655115608e-05, 3.0627427349827485e-06, 0.19719170033931732, 6.4298355937353335e-06, 3.444739604674396e-06, 0.23317119479179382, 0.0022796259727329016, 3.713432988661225e-06, 3.52880269929301e-05, 0.2685585916042328, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0004197956877760589, 0.00011805014219135046, 0.00014240501332096756, 3.796788223553449e-05, 0.00019043161591980606, 0.00176512252073735, 0.00057098304387182, 0.0005008855368942022, 8.840763621265069e-05, 0.0001420867774868384, 0.001663984265178442, 3.348111204104498e-05, 2.4413982828264125e-05, 0.00175465049687773, 6.520311580970883e-05, 2.414266964478884e-05, 0.0018299149814993143, 0.0015691040316596627, 3.974881110480055e-05, 0.00015712414460722357, 0.0018554475391283631, 0.9870065450668335, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [7.501777145080268e-05, 0.0014379826607182622, 6.345151632558554e-05, 0.00010864829528145492, 0.00015633167640771717, 3.2101231681735953e-06, 0.002203279873356223, 0.00022076554887462407, 5.240297468844801e-05, 4.8815789341460913e-05, 2.2644592263532104e-06, 1.5327233995776623e-05, 4.157140665483894e-06, 2.022839225901407e-06, 6.296796982496744e-06, 4.848669050261378e-05, 1.9928991150663933e-06, 3.247004497097805e-05, 0.0012695090845227242, 1.9632627299870364e-05, 1.8090934190695407e-06, 0.0005810288130305707, 0.9936450719833374, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [7.450151315424591e-05, 0.0006139386096037924, 0.0009361191187053919, 0.0008487795130349696, 0.002850631484761834, 1.0365051821281668e-05, 0.00021614256547763944, 0.00017397530609741807, 0.0020508402958512306, 0.005805297289043665, 8.055229955061805e-06, 8.086592424660921e-05, 0.0007702436414547265, 7.28818440620671e-06, 0.0010576159693300724, 0.002275596372783184, 6.663255135208601e-06, 0.00011621018347796053, 0.0005972451181150973, 8.736297604627907e-05, 6.332331849989714e-06, 6.096452852943912e-05, 6.090577517170459e-05, 0.9812840819358826, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0005848742439411581, 0.00015909200010355562, 1.083647021005163e-05, 7.365475903498009e-05, 0.00011349534906912595, 0.0008256662986241281, 0.00031911072437651455, 0.018529996275901794, 1.0226598533336073e-05, 4.9587219109525904e-05, 0.0007716424879617989, 4.454819281818345e-05, 9.865034371614456e-06, 0.0008067170856520534, 2.2673882995150052e-05, 1.2464139217627235e-05, 0.0008449103333987296, 0.008790111169219017, 3.5792407288681716e-05, 3.66286258213222e-05, 0.0008917527738958597, 0.0010791773675009608, 0.0003708462754730135, 0.0001083713723346591, 0.9654979109764099, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [1.6459925973322242e-05, 0.00017152438522316515, 3.2083211408462375e-05, 0.00010234172805212438, 0.002631882205605507, 9.886184670904186e-06, 3.250848749303259e-05, 3.7417539715534076e-05, 0.00012631539721041918, 4.9912112444872037e-05, 8.302540663862601e-06, 8.443430851912126e-05, 3.127968739136122e-05, 7.633363566128537e-06, 1.0101352927449625e-05, 5.6673809012863785e-05, 7.442129117407603e-06, 2.7689655325957574e-05, 1.841835728555452e-05, 2.879437261071871e-06, 6.840427886345424e-06, 4.279875156498747e-06, 0.00043176551116630435, 0.0001761750172590837, 8.995599637273699e-05, 0.9958257675170898, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [2.6368099497631192e-05, 0.00018008743063546717, 0.0001808296947274357, 0.00030464722658507526, 0.00039390946039929986, 4.674849333241582e-05, 2.7231719286646694e-05, 4.8734953452367336e-05, 0.00029131697374396026, 0.0004206168814562261, 3.804643711191602e-05, 0.0002524516312405467, 5.6067383411573246e-05, 3.82037615054287e-05, 0.0015365129802376032, 0.001253720954991877, 3.593428846215829e-05, 2.303666406078264e-05, 0.0001803626073524356, 0.0001226610183948651, 3.517790537443943e-05, 6.924685294507071e-05, 0.00011267305671935901, 0.0008507389575242996, 0.0001436190214008093, 0.00023528003657702357, 0.9930958151817322, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0007048255065456033, 5.675383727066219e-05, 3.351289251440903e-06, 7.472657671314664e-06, 1.785946005838923e-05, 0.0008156524272635579, 4.9529528041603044e-05, 0.0013530971482396126, 3.741759792319499e-05, 0.00014378006744664162, 0.0007639332907274365, 9.991676051868126e-06, 2.8844410735473502e-06, 0.0007552222232334316, 0.00010159592784475535, 3.1205443065118743e-06, 0.000806094438303262, 0.0011406756239011884, 1.4343090697366279e-05, 9.910167136695236e-06, 0.0008649186929687858, 8.663265907671303e-05, 3.4695403883233666e-05, 0.00010265821038046852, 0.007663471158593893, 4.187891317997128e-05, 3.4908698580693454e-06, 0.9844048619270325, 0.0, 0.0, 0.0, 0.0, 0.0], [2.0288332962081768e-05, 2.954272713395767e-05, 5.0375034334138036e-05, 0.00097783247474581, 0.3728252649307251, 6.6613192757358775e-06, 1.573364352225326e-05, 3.981243571615778e-05, 0.00022353240638040006, 0.00012674322351813316, 5.118445187690668e-06, 0.00024116165877785534, 1.2973839147889521e-05, 4.8006963879743125e-06, 2.3321663320530206e-05, 7.723527232883498e-05, 4.59344209957635e-06, 1.9647124645416625e-05, 0.00021129964443389326, 1.1453522347437683e-05, 4.380489826871781e-06, 1.4442671272263397e-05, 3.676743290270679e-05, 0.00011845333210658282, 3.7977195461280644e-05, 0.0007802762556821108, 0.00040483634802512825, 1.042955591401551e-05, 0.6236649751663208, 0.0, 0.0, 0.0, 0.0], [2.0582927390933037e-05, 8.831251761876047e-05, 0.00020454880723264068, 0.0003018901334144175, 8.223887562053278e-05, 1.9704415535670705e-05, 0.00014085126167628914, 2.896317164413631e-05, 7.669385013286956e-06, 3.7246169085847214e-05, 1.673677070357371e-05, 6.404696614481509e-05, 0.0006910591037012637, 1.6027584933908656e-05, 0.00015603694191668183, 0.00014825927792117, 1.5700084986747243e-05, 9.155390580417588e-05, 8.525098382961005e-05, 4.904517481918447e-06, 1.5784396964590997e-05, 5.293117283144966e-05, 0.000529835931956768, 0.0005658396985381842, 6.167324318084866e-05, 6.729497545165941e-05, 0.0003077442815992981, 1.0369159099354874e-05, 6.799342372687533e-05, 0.9960988759994507, 0.0, 0.0, 0.0], [1.3371318345889449e-05, 0.0009821535786613822, 0.0004154810740146786, 0.0001144233756349422, 0.00038730789674445987, 5.660860551870428e-06, 0.0012746280990540981, 0.0005708065000362694, 0.0006383630097843707, 0.0005776663310825825, 4.127733518544119e-06, 9.16175213205861e-06, 9.142841736320406e-05, 3.774867764150258e-06, 1.3575321645475924e-05, 0.0002916179655585438, 3.4474890071578557e-06, 7.899192860350013e-05, 0.003189122537150979, 4.885083853878314e-06, 3.165286670991918e-06, 1.4087469025980681e-05, 0.0001567144354339689, 0.0003544889041222632, 0.00017265471979044378, 0.0013050634879618883, 0.00021867688337806612, 2.6776719096233137e-05, 0.00026460207300260663, 1.3334529285202734e-05, 0.9888005256652832, 0.0, 0.0], [2.745506208157167e-05, 0.00016438262537121773, 7.99642366473563e-05, 0.001191496616229415, 0.0007883630460128188, 2.658414359757444e-06, 3.005756479979027e-05, 7.4579688771336805e-06, 0.00014940995606593788, 2.8857133656856604e-05, 1.873827500276093e-06, 0.00033288003760389984, 5.160855653230101e-05, 1.7577448261363315e-06, 0.0001265659084310755, 0.00014267012011259794, 1.6954454622464254e-06, 2.1952573661110364e-05, 0.00023040804080665112, 4.429338878253475e-05, 1.6103306279546814e-06, 2.7008050892618485e-05, 0.00023884844267740846, 0.00019046018132939935, 9.496136954112444e-06, 0.000446643796749413, 0.00022095769236329943, 5.379170943342615e-06, 0.0006956399301998317, 0.0001547076681163162, 0.0002548544143792242, 0.9943286776542664, 0.0], [0.006231049541383982, 9.72282505244948e-05, 6.871603090985445e-06, 2.1151156033738516e-05, 5.8280591474613175e-05, 0.007238905411213636, 2.0987936295568943e-05, 0.00025459096650592983, 6.243876850930974e-05, 2.0924684577039443e-05, 0.007872147485613823, 5.5853353842394426e-05, 9.868796041700989e-06, 0.009169402532279491, 7.203016866696998e-05, 7.068700597301358e-06, 0.010345976799726486, 0.0013096537441015244, 3.803684376180172e-05, 8.022711699595675e-05, 0.012053261511027813, 4.071998773724772e-05, 3.6860749332845444e-06, 3.4713455534074455e-05, 0.0005061827832832932, 8.918941603042185e-05, 2.9112143238307908e-05, 0.0012772815534844995, 8.489656465826556e-05, 0.00018447409092914313, 0.00013425754150375724, 6.813067739130929e-05, 0.9425214529037476]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.943029522895813, 0.05697042867541313, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9185556173324585, 0.03280005604028702, 0.048644352704286575, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8779287934303284, 0.056434255093336105, 0.04271192103624344, 0.02292500250041485, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.804131805896759, 0.029098201543092728, 0.075567327439785, 0.05643591657280922, 0.03476677089929581, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4943104684352875, 0.02018355205655098, 0.027966560795903206, 0.01831907220184803, 0.03144203498959541, 0.40777823328971863, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6057478785514832, 0.02924242615699768, 0.09491512179374695, 0.07609347254037857, 0.06614658236503601, 0.08705786615610123, 0.0407966710627079, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4483822286128998, 0.04542430862784386, 0.0740148276090622, 0.06864846497774124, 0.09376619011163712, 0.0877426490187645, 0.06534270942211151, 0.11667861044406891, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.49169260263442993, 0.13782073557376862, 0.03955018147826195, 0.06153320148587227, 0.04539964720606804, 0.04073145240545273, 0.06228705495595932, 0.05861866846680641, 0.062366463243961334, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5404124855995178, 0.044426653534173965, 0.03957854211330414, 0.04188809171319008, 0.07529856264591217, 0.04669506475329399, 0.048475231975317, 0.05500520393252373, 0.08293062448501587, 0.02528950944542885, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.28273555636405945, 0.014234175905585289, 0.017647748813033104, 0.011433064006268978, 0.021741706877946854, 0.2666538953781128, 0.015403537079691887, 0.047349266707897186, 0.017767544835805893, 0.013926065526902676, 0.2911074459552765, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.34496626257896423, 0.046116773039102554, 0.05771546810865402, 0.11131857335567474, 0.11289366334676743, 0.027930336073040962, 0.0385919027030468, 0.05656527727842331, 0.05864058807492256, 0.06648595631122589, 0.026114359498023987, 0.052660852670669556, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.46992170810699463, 0.032015684992074966, 0.1077289879322052, 0.027006765827536583, 0.04465881362557411, 0.022773560136556625, 0.0231170691549778, 0.025491517037153244, 0.049502693116664886, 0.026573937386274338, 0.01970885880291462, 0.06337957829236984, 0.08812081813812256, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.20661765336990356, 0.01037929393351078, 0.012261927127838135, 0.008319098502397537, 0.016007069498300552, 0.19527742266654968, 0.011453290469944477, 0.03475669026374817, 0.01307358592748642, 0.010938976891338825, 0.21602483093738556, 0.005866494961082935, 0.023642312735319138, 0.23538129031658173, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.33993491530418396, 0.029161635786294937, 0.09540709853172302, 0.03395187482237816, 0.08440458029508591, 0.012559544295072556, 0.02935866080224514, 0.024564165621995926, 0.10622433573007584, 0.04689216986298561, 0.011469592340290546, 0.006369193084537983, 0.11145274341106415, 0.011317990720272064, 0.0569315105676651, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4040880799293518, 0.024195658043026924, 0.038910072296857834, 0.014727434143424034, 0.024456558749079704, 0.038450006395578384, 0.03923036903142929, 0.037171367555856705, 0.06030004844069481, 0.041985444724559784, 0.037167176604270935, 0.01639123260974884, 0.0392896942794323, 0.03772977367043495, 0.13448578119277954, 0.011421292088925838, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15876448154449463, 0.008203906007111073, 0.0092952661216259, 0.0062217190861701965, 0.011793204583227634, 0.15069791674613953, 0.008851958438754082, 0.026313532143831253, 0.010186923667788506, 0.00843372568488121, 0.16762229800224304, 0.004420033656060696, 0.01821179874241352, 0.183508038520813, 0.020927347242832184, 0.006447790190577507, 0.20010006427764893, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.17539216578006744, 0.02056964486837387, 0.01829143613576889, 0.009298020042479038, 0.017377890646457672, 0.042534783482551575, 0.02070159651339054, 0.050443943589925766, 0.02543804794549942, 0.017218226566910744, 0.04311535507440567, 0.013349834829568863, 0.0285286046564579, 0.045972540974617004, 0.03408820927143097, 0.019834214821457863, 0.04992839694023132, 0.36791715025901794, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.26033347845077515, 0.017148371785879135, 0.03745276480913162, 0.07594798505306244, 0.04674699530005455, 0.018068527802824974, 0.031346458941698074, 0.037415146827697754, 0.07175809890031815, 0.05872536823153496, 0.0170787014067173, 0.04030593857169151, 0.05706355720758438, 0.01711357943713665, 0.1049133837223053, 0.046705346554517746, 0.017230207100510597, 0.024682143703103065, 0.019963975995779037, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.30547064542770386, 0.05190496891736984, 0.043468981981277466, 0.02184685505926609, 0.02101718634366989, 0.03390473872423172, 0.04190470278263092, 0.03909286484122276, 0.028871973976492882, 0.023003432899713516, 0.03205801919102669, 0.023334519937634468, 0.0711059421300888, 0.03290088474750519, 0.061641961336135864, 0.03183262050151825, 0.033767636865377426, 0.04571487754583359, 0.035015225410461426, 0.022141898050904274, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1224295049905777, 0.006105300039052963, 0.006670300383120775, 0.00458177737891674, 0.009337784722447395, 0.11381794512271881, 0.006783359684050083, 0.01971990056335926, 0.00758085772395134, 0.006613647099584341, 0.12765924632549286, 0.0035026604309678078, 0.01400233618915081, 0.14000985026359558, 0.015684885904192924, 0.005092614330351353, 0.15336214005947113, 0.03527415543794632, 0.02246563322842121, 0.006954458076506853, 0.17235167324543, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.20538486540317535, 0.03477850183844566, 0.014682911336421967, 0.03118330053985119, 0.030931269749999046, 0.021952766925096512, 0.03290865570306778, 0.05740531533956528, 0.05587516725063324, 0.04864277318120003, 0.02352074719965458, 0.015108555555343628, 0.02738633006811142, 0.02451845072209835, 0.060604583472013474, 0.034776147454977036, 0.026137804612517357, 0.051684360951185226, 0.06281405687332153, 0.020291464403271675, 0.02860172651708126, 0.0908101499080658, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.23978599905967712, 0.031323499977588654, 0.05037755146622658, 0.01586942747235298, 0.03901456296443939, 0.02280554547905922, 0.04285356029868126, 0.028882304206490517, 0.04046262055635452, 0.0341072604060173, 0.022644521668553352, 0.03923071548342705, 0.0723857581615448, 0.022345518693327904, 0.04958106949925423, 0.03193334490060806, 0.023325597867369652, 0.04521360620856285, 0.030554356053471565, 0.022876497358083725, 0.02473229542374611, 0.055494822561740875, 0.014199569821357727, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.23581692576408386, 0.020558306947350502, 0.04375005513429642, 0.02970486506819725, 0.03703876957297325, 0.014953454956412315, 0.04004311189055443, 0.027184365317225456, 0.0457618422806263, 0.03809259459376335, 0.014181883074343204, 0.03789152577519417, 0.06518243253231049, 0.014182924292981625, 0.05489495024085045, 0.023720961064100266, 0.014592787250876427, 0.025570029392838478, 0.07356183975934982, 0.039182331413030624, 0.014925522729754448, 0.04628865420818329, 0.027801064774394035, 0.015118853189051151, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14254964888095856, 0.012010160833597183, 0.016881300136446953, 0.020742561668157578, 0.032451752573251724, 0.029625998809933662, 0.030295221135020256, 0.05620869994163513, 0.029608748853206635, 0.02948184125125408, 0.03263983502984047, 0.010038234293460846, 0.04078620672225952, 0.03462785482406616, 0.03391636535525322, 0.02015574835240841, 0.036843158304691315, 0.0606469102203846, 0.047447122633457184, 0.032525286078453064, 0.040400370955467224, 0.05947763845324516, 0.03129400685429573, 0.04792547971010208, 0.07141980528831482, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2222907692193985, 0.059111371636390686, 0.03702675551176071, 0.04059014096856117, 0.0272544976323843, 0.019174763932824135, 0.03171538934111595, 0.02046213671565056, 0.038113273680210114, 0.019927890971302986, 0.018538912758231163, 0.015436704270541668, 0.04536491632461548, 0.01935577020049095, 0.050358302891254425, 0.03328138589859009, 0.02017974853515625, 0.03679436072707176, 0.04331387206912041, 0.028476405888795853, 0.02131732925772667, 0.047712888568639755, 0.013107037171721458, 0.02633604407310486, 0.030211349949240685, 0.03454800695180893, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.28438133001327515, 0.023870104923844337, 0.04641882702708244, 0.010260095819830894, 0.03390985727310181, 0.01830146834254265, 0.0237088892608881, 0.0247668344527483, 0.025767752900719643, 0.022968804463744164, 0.016735462471842766, 0.013406947255134583, 0.045986607670784, 0.01667322963476181, 0.08106391876935959, 0.05033260956406593, 0.016726160421967506, 0.019904790446162224, 0.03032534383237362, 0.01014632172882557, 0.01731823943555355, 0.019040953367948532, 0.01108111348003149, 0.052046384662389755, 0.033353518694639206, 0.03880883380770683, 0.012695519253611565, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11854357272386551, 0.01335094589740038, 0.013422034680843353, 0.030273202806711197, 0.02616293355822563, 0.020776210352778435, 0.021280204877257347, 0.03798932954668999, 0.03536444157361984, 0.036651790142059326, 0.022004006430506706, 0.01538003608584404, 0.030023131519556046, 0.023299671709537506, 0.03000757284462452, 0.014861369505524635, 0.02452193759381771, 0.03999800235033035, 0.03900156170129776, 0.036007679998874664, 0.026851212605834007, 0.06795871257781982, 0.038931310176849365, 0.057258062064647675, 0.0665983259677887, 0.03075685352087021, 0.023343220353126526, 0.059382691979408264, 0.0, 0.0, 0.0, 0.0, 0.0], [0.25091853737831116, 0.01135188713669777, 0.023033952340483665, 0.01884976588189602, 0.013208821415901184, 0.016430404037237167, 0.03763698413968086, 0.021519413217902184, 0.03823148459196091, 0.03122309409081936, 0.01608109660446644, 0.017179012298583984, 0.0823964774608612, 0.01577541045844555, 0.04969313368201256, 0.033979032188653946, 0.016469145193696022, 0.025656109675765038, 0.053269319236278534, 0.024380972608923912, 0.017032906413078308, 0.031062550842761993, 0.014834268018603325, 0.04310780018568039, 0.027818351984024048, 0.015127941034734249, 0.012749841436743736, 0.026852674782276154, 0.014129597693681717, 0.0, 0.0, 0.0, 0.0], [0.19256946444511414, 0.022833675146102905, 0.01449588406831026, 0.028055010363459587, 0.03290426358580589, 0.018577100709080696, 0.023776723071932793, 0.014988909475505352, 0.027755476534366608, 0.01995212957262993, 0.018426887691020966, 0.0268084853887558, 0.04026195779442787, 0.01895732805132866, 0.01998024806380272, 0.039050325751304626, 0.01949433982372284, 0.030714301392436028, 0.07932323962450027, 0.03619766980409622, 0.02037958987057209, 0.023319289088249207, 0.018723847344517708, 0.05692768841981888, 0.023927535861730576, 0.03924323245882988, 0.02178351581096649, 0.020374197512865067, 0.04178833216428757, 0.00840924121439457, 0.0, 0.0, 0.0], [0.1891939342021942, 0.01435206737369299, 0.027829350903630257, 0.0189360398799181, 0.05455230921506882, 0.024302387610077858, 0.0207525584846735, 0.03050178475677967, 0.01690041646361351, 0.029904041439294815, 0.023111775517463684, 0.02166486158967018, 0.03335980698466301, 0.023050500079989433, 0.02792702615261078, 0.026253484189510345, 0.0242579597979784, 0.024394970387220383, 0.017666669562458992, 0.02208106406033039, 0.025441503152251244, 0.023996729403734207, 0.015941407531499863, 0.021863479167222977, 0.04876275360584259, 0.01023405697196722, 0.025762980803847313, 0.06192327290773392, 0.06737110018730164, 0.01818411611020565, 0.009525515139102936, 0.0, 0.0], [0.27111950516700745, 0.05460572615265846, 0.039705973118543625, 0.03512895852327347, 0.02031376212835312, 0.0081838583573699, 0.022556638345122337, 0.01403643935918808, 0.02619846910238266, 0.032355356961488724, 0.007185023743659258, 0.010813490487635136, 0.05162610113620758, 0.007057540584355593, 0.04232776537537575, 0.01850598305463791, 0.006952997762709856, 0.012550847604870796, 0.037855181843042374, 0.014787373133003712, 0.007130768615752459, 0.01895672082901001, 0.01075897179543972, 0.023540528491139412, 0.01625620573759079, 0.007915404625236988, 0.03652312234044075, 0.014562270604074001, 0.019251469522714615, 0.010911807417869568, 0.0343967042863369, 0.06592899560928345, 0.0], [0.14836537837982178, 0.014506888575851917, 0.007646295242011547, 0.011038758791983128, 0.02630247175693512, 0.01319483295083046, 0.027184898033738136, 0.017392203211784363, 0.016122139990329742, 0.032880790531635284, 0.014050280675292015, 0.0071673463098704815, 0.016120055690407753, 0.014647023752331734, 0.018450895324349403, 0.013102126307785511, 0.01528322696685791, 0.01454298384487629, 0.05961218476295471, 0.02238387055695057, 0.016796991229057312, 0.10928259789943695, 0.07264000922441483, 0.08393709361553192, 0.025248989462852478, 0.03774847462773323, 0.016226941719651222, 0.024953871965408325, 0.042165856808423996, 0.004525808617472649, 0.014491353183984756, 0.01914571039378643, 0.022841647267341614]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09646990895271301, 0.9035300612449646, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04325238987803459, 0.08177754282951355, 0.8749701380729675, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09995394200086594, 0.025312727317214012, 0.02010803483426571, 0.8546252846717834, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.024889368563890457, 0.00320735527202487, 0.0018421593122184277, 0.022361503913998604, 0.9476996660232544, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10732383280992508, 0.017841672524809837, 0.01955333538353443, 0.04333319514989853, 0.10211498290300369, 0.7098329067230225, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.006426361855119467, 0.00044797800364904106, 0.00014756589371245354, 0.00046936701983213425, 0.0014411886222660542, 0.003859696676954627, 0.98720782995224, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00103545852471143, 0.00019901820633094758, 0.00016020411567296833, 6.937271973583847e-05, 0.00038674141978845, 0.005171590484678745, 0.8964057564735413, 0.0965719074010849, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0012883321614935994, 0.0003233453317079693, 0.00026527189766056836, 0.00025490616098977625, 0.000201298258616589, 0.00010049015691038221, 0.0005700886249542236, 0.000409130152547732, 0.9965871572494507, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0001874157169368118, 1.6196530850720592e-05, 4.2281039895897266e-06, 0.000287588540231809, 1.1125704986625351e-05, 9.805656191019807e-06, 0.0001556719362270087, 7.632971392013133e-05, 0.0034869094379246235, 0.9957647323608398, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015741512179374695, 0.0006393603980541229, 0.00045715278247371316, 0.0009912519017234445, 0.0021140354219824076, 0.0180897768586874, 0.0471203438937664, 0.07010912150144577, 0.06152832508087158, 0.27690398693084717, 0.5063051581382751, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0006166594685055315, 0.0005207078065723181, 4.61151976196561e-05, 0.00146130018401891, 0.0005623754695989192, 4.4476037146523595e-05, 0.00036539664142765105, 0.0002860166132450104, 0.004506127443164587, 0.005816521123051643, 0.0007244544103741646, 0.9850499033927917, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0010478557087481022, 3.062814721488394e-05, 0.00017344979278277606, 5.423119000624865e-05, 6.388417386915535e-05, 1.1261126928729936e-05, 1.7169008060591295e-05, 1.3931307876191568e-05, 0.0020760390907526016, 0.00026926834834739566, 0.00015268517017830163, 0.0036844322457909584, 0.992405116558075, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.010746272280812263, 0.00025354334502480924, 0.0001532292371848598, 0.00029287015786394477, 0.0005376793560571969, 0.004423544742166996, 0.00988433975726366, 0.012843050062656403, 0.012738612480461597, 0.05966852977871895, 0.107729971408844, 0.024745700880885124, 0.07808694988489151, 0.6778956651687622, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00041218000114895403, 2.6218738639727235e-05, 1.7755904991645366e-05, 0.00019181883544661105, 3.2979492061713245e-06, 3.912682132067857e-06, 1.043809788825456e-05, 4.9068494263337925e-06, 0.0005868570297025144, 0.003038151189684868, 3.693124745041132e-05, 0.0007724304450675845, 0.009622604586184025, 0.00016094396414700896, 0.9851114153862, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00010886583186220378, 3.5845487218466587e-06, 5.191657237446634e-06, 3.083837145823054e-05, 1.8408762116450816e-05, 7.954765806061914e-07, 3.3532073757669423e-06, 6.574365215783473e-06, 0.0007270933128893375, 0.0018232465954497457, 7.853259376133792e-06, 0.0003062748000957072, 0.006975048687309027, 3.4850869269575924e-05, 0.0282927006483078, 0.9616552591323853, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.006073986645787954, 8.815194451017305e-05, 4.5837816287530586e-05, 8.250484825111926e-05, 0.0001272043155040592, 0.0009312513284385204, 0.0016699967673048377, 0.001938193803653121, 0.002101697726175189, 0.010131671093404293, 0.016047311946749687, 0.003835388459265232, 0.012242135591804981, 0.10037057846784592, 0.07853133976459503, 0.10997355729341507, 0.655809223651886, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0021862962748855352, 1.1892905604327098e-05, 1.673595943429973e-05, 2.081541060761083e-05, 2.116251016559545e-05, 0.000771492428611964, 0.0008652104297652841, 0.0005560291465371847, 0.0001565588463563472, 0.0019499900517985225, 0.010891195386648178, 0.0003954307467211038, 0.0015144682256504893, 0.06916307657957077, 0.00430810172110796, 0.004035161342471838, 0.4939703941345215, 0.40916600823402405, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [1.979802800633479e-05, 4.972388865098765e-07, 9.16571263331889e-09, 1.3230416584519844e-07, 5.6018219396491986e-08, 1.1903876995233986e-08, 6.153105118755775e-07, 4.0685506519366754e-08, 4.517455181485275e-06, 1.4017791727383155e-05, 1.6456915830076468e-07, 1.086384372683824e-06, 7.44406133890152e-06, 1.005667400022503e-06, 2.75950224022381e-05, 0.00031378038693219423, 6.201125415827846e-06, 8.213370165321976e-06, 0.9995948672294617, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [6.497167760244338e-06, 8.692019264344708e-07, 1.4051378229851252e-06, 3.7704828059759166e-07, 3.3291723866568645e-07, 6.104961869368708e-08, 2.542995503063139e-08, 1.1067748317827864e-07, 3.670873775263317e-05, 5.704808927475824e-07, 6.172335247356386e-07, 0.00015627949323970824, 0.00014737885794602334, 3.010375849044067e-06, 6.130654219305143e-05, 0.0017769918777048588, 1.8545919374446385e-05, 3.426316106924787e-05, 0.923351526260376, 0.07440318167209625, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.003649001009762287, 3.7494322896236554e-05, 1.74708020495018e-05, 2.8449197998270392e-05, 3.312424087198451e-05, 0.00019569217693060637, 0.0002647593501023948, 0.0002469043538440019, 0.00028033158741891384, 0.0013949184212833643, 0.0017910305177792907, 0.00042468024184927344, 0.001462370972149074, 0.009717755019664764, 0.008053947240114212, 0.01320058386772871, 0.06315966695547104, 0.07152792811393738, 0.07523112744092941, 0.024205094203352928, 0.7250776290893555, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.000641089805867523, 3.925701548723737e-06, 1.1607372698563267e-06, 1.537548200758465e-06, 9.072650755115319e-07, 8.05819763627369e-06, 1.3223971109255217e-05, 4.6199284042813815e-06, 3.678836583276279e-05, 4.5651795517187566e-05, 7.577823998872191e-05, 2.274175494676456e-05, 5.6684530136408284e-05, 0.0004737511626444757, 0.0003640766954049468, 0.00042400247184559703, 0.00375733501277864, 0.004363126587122679, 0.007395228371024132, 0.005022629629820585, 0.05575620383024216, 0.9215314984321594, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0005377684719860554, 8.12255109394755e-07, 2.3018603769742185e-06, 1.470038341722102e-06, 1.8703678961173864e-06, 6.02451621034561e-07, 4.12783674619277e-06, 1.6705761254343088e-06, 2.047191856036079e-06, 7.430045661749318e-05, 3.5942291560786543e-06, 7.171521247073542e-06, 3.052820102311671e-05, 1.8927734345197678e-05, 0.0002543879672884941, 0.0001949639990925789, 0.00012626573152374476, 0.00018230534624308348, 0.0037352300714701414, 0.0006727128056809306, 0.0015798343811184168, 0.011177700012922287, 0.9813893437385559, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [8.512814383720979e-05, 4.870869929618493e-07, 1.2458942819648655e-06, 2.0811235401652084e-07, 1.92428601053507e-07, 3.704030859807972e-08, 4.5552428673545364e-07, 5.504545086409962e-08, 1.532036321805208e-06, 5.083304131403565e-06, 2.1400977345820138e-07, 1.459531858927221e-06, 1.4768386790819932e-05, 1.0635022817950812e-06, 2.9461503800121136e-05, 5.7288212701678276e-05, 6.615699021494947e-06, 7.844223546271678e-06, 0.00015063839964568615, 0.00022636978246737272, 7.973578613018617e-05, 0.00013040869089309126, 0.010868428274989128, 0.988331139087677, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0004166325961705297, 2.250373881906853e-06, 5.27054771737312e-06, 2.7549118385650218e-05, 3.963056951761246e-05, 7.514749540860066e-06, 8.10094024927821e-06, 1.1581340913835447e-05, 1.320597675658064e-05, 6.750728789484128e-05, 2.5212739274138585e-05, 1.486069868406048e-05, 7.521701627410948e-05, 0.00011457462824182585, 0.0002574531245045364, 0.0003666863194666803, 0.0006904753972776234, 0.0014518487732857466, 0.0032460209913551807, 0.0006353403441607952, 0.008745575323700905, 0.015190862119197845, 0.01815020479261875, 0.1330331563949585, 0.8174033164978027, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [1.673920633038506e-05, 8.245287830277448e-08, 3.257373748510872e-08, 2.873728810470766e-08, 1.6217634879467369e-07, 1.5275355425004022e-09, 2.3278557748085404e-08, 5.576718198341268e-09, 1.2825742601307866e-07, 1.3429436762635305e-07, 6.160537857624604e-09, 2.5232478151338e-07, 3.2057448606792605e-06, 2.6277179543399143e-08, 7.619933626301645e-07, 7.901945537014399e-06, 1.3940400833689637e-07, 1.955255868324457e-07, 0.00032022525556385517, 8.173647074727342e-07, 1.562996089887747e-06, 3.763153017644072e-06, 0.0007450500270351768, 0.0037152147851884365, 3.934417327400297e-05, 0.9951443076133728, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00023842290102038532, 5.040124960942194e-06, 1.3208604059400386e-06, 8.599836291978136e-06, 5.166078608453972e-06, 1.3470925352976337e-07, 6.710816364829952e-07, 2.699698029573483e-07, 6.345731435430935e-06, 4.417831223690882e-05, 2.7444934858067427e-07, 2.017119368247222e-05, 1.652665923757013e-05, 8.122791541609331e-07, 0.0001077100241673179, 0.0001239744306076318, 3.5075922824034933e-06, 8.505840924044605e-06, 6.980268517509103e-05, 0.0002681369078345597, 2.8792866942239925e-05, 0.00011957906826864928, 0.0001296169066336006, 0.007425930816680193, 0.0007109709549695253, 0.005815016105771065, 0.9848405718803406, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00040899505256675184, 6.095365279179532e-07, 1.9963911199738504e-06, 9.31032991502434e-06, 3.769046315937885e-06, 1.987656105484348e-06, 4.57654266483587e-07, 1.069074301085493e-06, 3.087174036409124e-06, 9.211397809849586e-06, 4.724369318864774e-06, 1.2070032653355156e-06, 5.095620508654974e-06, 1.8047117919195443e-05, 4.857865133089945e-05, 4.169386738794856e-05, 0.00010537513298913836, 0.00022466867812909186, 0.00011528613686095923, 0.0002492894127499312, 0.0013673142530024052, 0.0016190343303605914, 0.0013275155797600746, 0.003491780487820506, 0.06305663287639618, 0.00646906066685915, 0.0951264277100563, 0.8262877464294434, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00031804729951545596, 1.4181842971083825e-06, 3.1855009297032666e-07, 2.7823105028801365e-06, 5.388588397181593e-05, 4.300602540752152e-08, 5.386270913732005e-07, 8.497841008647811e-08, 5.99971485826245e-07, 4.110329882678343e-06, 7.099399113030813e-08, 8.454308044747449e-07, 2.5266151624236954e-06, 2.198386539475905e-07, 6.098252015362959e-06, 2.1130046661710367e-05, 9.411802466274821e-07, 9.95040522866475e-07, 3.041357376787346e-05, 2.976995119752246e-06, 8.453868758806493e-06, 2.6007013730122708e-05, 0.00023055852216202766, 0.0013383383629843593, 0.0002175826084567234, 0.007014401722699404, 0.017924025654792786, 0.0005302676581777632, 0.9722622632980347, 0.0, 0.0, 0.0, 0.0], [1.8505332377571904e-07, 5.565692351439111e-09, 7.262683787701008e-10, 7.745998509278706e-09, 1.507951097323712e-08, 8.22852175286215e-10, 1.7915348093922034e-09, 1.7345647140842857e-09, 1.0962556196147943e-08, 5.0032749498996054e-08, 3.9228162940219136e-09, 1.0013924622853665e-07, 2.333774595797422e-08, 1.6555254944705666e-08, 5.1713197279923406e-08, 1.614605764643784e-07, 9.827901692460728e-08, 2.707449482386437e-07, 5.411987331171986e-06, 2.5083886612264905e-07, 1.1805956319221877e-06, 2.8004205887555145e-06, 4.61038098364952e-06, 0.00022679210815113038, 5.7942423154599965e-05, 0.0004519550711847842, 0.0001197646779473871, 0.00023980921832844615, 0.001874006469734013, 0.9970145225524902, 0.0, 0.0, 0.0], [4.347462163423188e-05, 2.968874923681142e-06, 3.919368793958711e-07, 9.117065360442211e-07, 4.364489711861097e-07, 4.686591115188321e-09, 9.240378062713717e-08, 1.2419349992853768e-08, 7.188964445958845e-08, 6.304543944679608e-07, 6.108888950251412e-09, 1.9987160726486763e-07, 1.4578674836229766e-06, 1.4727343788933922e-08, 1.7957326292616926e-07, 2.6422712835483253e-05, 5.112656609185251e-08, 4.710674161856332e-08, 0.0015761180547997355, 2.198941047026892e-06, 3.3339856031489035e-07, 2.1814876163261943e-06, 0.00016240392869804054, 0.0017777654575183988, 1.1749207260436378e-05, 0.0036160903982818127, 6.199482595548034e-05, 3.5168231988791376e-05, 0.001148868934251368, 0.016085581853985786, 0.9754422307014465, 0.0, 0.0], [0.0011562927393242717, 6.199047675181646e-06, 6.137794912319805e-07, 3.102660230069887e-06, 8.228408319155278e-07, 4.3660193682626414e-08, 9.36325577072239e-08, 2.7735543639550997e-08, 5.109529865876539e-07, 1.2355309308986762e-06, 3.8253045175906664e-08, 1.127230689235148e-06, 1.9008897425010218e-06, 8.714057031511402e-08, 5.398183020588476e-06, 7.393444320769049e-06, 3.128881189695676e-07, 3.0146313179102435e-07, 0.00013595378550235182, 1.3343835234991275e-05, 2.39964992942987e-06, 3.871273293043487e-05, 0.0008219636511057615, 0.0006131280679255724, 2.151841727027204e-05, 0.004140730947256088, 0.0008760862983763218, 0.00011684626952046528, 0.0021590692922472954, 0.027525268495082855, 0.04434971883893013, 0.9179997444152832, 0.0], [0.003194290678948164, 4.681676728068851e-05, 3.49913134414237e-05, 2.7043113732361235e-05, 1.4000337614561431e-05, 1.7085165382013656e-05, 8.051893928495701e-06, 3.909831775672501e-06, 1.0716959877754562e-05, 1.793081537471153e-05, 1.4207512322172988e-05, 2.691133522603195e-05, 2.6411998987896368e-05, 3.1100229534786195e-05, 1.1198581887583714e-05, 5.485505607794039e-05, 0.00011474490747787058, 0.00010493675654288381, 0.00028063071658834815, 0.00020839076023548841, 0.0009329256135970354, 0.0018990016542375088, 0.0011267587542533875, 0.0019257268868386745, 0.0039171562530100346, 0.007499460596591234, 0.010695938020944595, 0.023199880495667458, 0.03039817325770855, 0.12729351222515106, 0.04325327277183533, 0.1766805499792099, 0.5669293999671936]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.25308629870414734, 0.746913731098175, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.30671578645706177, 0.32906386256217957, 0.3642203211784363, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07416975498199463, 0.1618966907262802, 0.054325949400663376, 0.7096075415611267, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1668822318315506, 0.039017826318740845, 0.03822460398077965, 0.21398353576660156, 0.5418918132781982, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19483636319637299, 0.2111925631761551, 0.05150565132498741, 0.0870383232831955, 0.2299954891204834, 0.22543159127235413, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13900135457515717, 0.02974863536655903, 0.03860695660114288, 0.05133272334933281, 0.19284246861934662, 0.08012373745441437, 0.4683440625667572, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08969778567552567, 0.04080599546432495, 0.03473307564854622, 0.08414525538682938, 0.09911047667264938, 0.07059449702501297, 0.1361657828092575, 0.4447471499443054, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05986519902944565, 0.020190449431538582, 0.018785323947668076, 0.1058472990989685, 0.057948824018239975, 0.027517711743712425, 0.0566631518304348, 0.08226760476827621, 0.5709145069122314, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.010379260405898094, 0.0046982537023723125, 0.004143984522670507, 0.0072914063930511475, 0.006256693042814732, 0.0033180469181388617, 0.009175012819468975, 0.018754413351416588, 0.03393147885799408, 0.9020513892173767, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07270114123821259, 0.0477648489177227, 0.010500345379114151, 0.0161594245582819, 0.046178195625543594, 0.05249456688761711, 0.051845718175172806, 0.21189512312412262, 0.035158202052116394, 0.17267179489135742, 0.2826306223869324, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015126891434192657, 0.004731189925223589, 0.0023226316552609205, 0.0065757399424910545, 0.01836245507001877, 0.003339666873216629, 0.008784224279224873, 0.007409745827317238, 0.006289808079600334, 0.07638943940401077, 0.01200348511338234, 0.838664710521698, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.03233193978667259, 0.010628965683281422, 0.0026151640340685844, 0.0011762931244447827, 0.0030932866502553225, 0.0015055524418130517, 0.007079718168824911, 0.00283446884714067, 0.005003884434700012, 0.012032588012516499, 0.0039877742528915405, 0.051872409880161285, 0.8658380508422852, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05045590549707413, 0.02653883770108223, 0.005783271975815296, 0.00808729324489832, 0.021799379959702492, 0.025670334696769714, 0.022779036313295364, 0.09139905869960785, 0.01501737255603075, 0.07093921303749084, 0.12448635697364807, 0.1266833245754242, 0.0610213503241539, 0.34933918714523315, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05474122613668442, 0.05296846479177475, 0.00398812722414732, 0.012351620942354202, 0.00441562058404088, 0.0035962786059826612, 0.011385707184672356, 0.009828065522015095, 0.014749741181731224, 0.07078825682401657, 0.011209187097847462, 0.05164055898785591, 0.1973438560962677, 0.027129262685775757, 0.4738638997077942, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.007163074798882008, 0.004963848739862442, 0.0027692278381437063, 0.0019424431957304478, 0.010544748045504093, 0.0014144877204671502, 0.003663665847852826, 0.003149948548525572, 0.005481299012899399, 0.021614043042063713, 0.003922200761735439, 0.07935082912445068, 0.2247791737318039, 0.009166955016553402, 0.027402665466070175, 0.5926714539527893, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.034611064940690994, 0.015296351164579391, 0.003095126012340188, 0.0038990958128124475, 0.009989401325583458, 0.01144686620682478, 0.009189358912408352, 0.03356517106294632, 0.005761212203651667, 0.025581011548638344, 0.043846867978572845, 0.04521361365914345, 0.02055339701473713, 0.1196233481168747, 0.1567324846982956, 0.13173452019691467, 0.32986119389533997, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.017253872007131577, 0.004397525917738676, 0.004968690220266581, 0.007064312230795622, 0.006634894758462906, 0.006910949479788542, 0.011386900208890438, 0.016178512945771217, 0.020317763090133667, 0.020396802574396133, 0.02342209964990616, 0.01890912838280201, 0.04024204611778259, 0.06200675293803215, 0.116453617811203, 0.07975102961063385, 0.16864468157291412, 0.37506037950515747, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.002459116280078888, 0.00024919764837250113, 3.400466084713116e-05, 0.0002113294176524505, 0.00020066798606421798, 0.0001258924457943067, 0.00036506837932392955, 0.00037359801353886724, 0.00013798549480270594, 0.0004702212172560394, 0.00027151827816851437, 0.003417074913159013, 0.0006049809162504971, 0.0005454609636217356, 0.0008937679813243449, 0.0015852133510634303, 0.0012150758411735296, 0.00282126828096807, 0.9840186238288879, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00015282434469554573, 0.038923632353544235, 0.00047151994658634067, 0.0003731817996595055, 3.271793684689328e-05, 1.063560375769157e-05, 3.8245766518230084e-06, 1.3244694855529815e-05, 6.418924022000283e-05, 2.7344345653546043e-05, 2.8319956982159056e-05, 0.0022295680828392506, 0.0013866383815184236, 7.163518603192642e-05, 0.0001402555499225855, 0.00824644137173891, 0.00019678636454045773, 0.00020546668383758515, 0.8976381421089172, 0.049783725291490555, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02645810693502426, 0.010410889983177185, 0.0021332555916160345, 0.0024560203310102224, 0.005542597267776728, 0.005954570136964321, 0.0041958331130445, 0.013451533392071724, 0.0023519538808614016, 0.009471958503127098, 0.015433433465659618, 0.014804964885115623, 0.006865760777145624, 0.037873148918151855, 0.047449350357055664, 0.045472823083400726, 0.10199166089296341, 0.08666391670703888, 0.06470184028148651, 0.10160762816667557, 0.3947088122367859, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.021157892420887947, 0.0035484335385262966, 0.001744111767038703, 0.0035093079786747694, 0.004201894626021385, 0.00257777888327837, 0.004015856888145208, 0.004598999861627817, 0.003955156076699495, 0.006455833092331886, 0.00590789457783103, 0.0037331636995077133, 0.009013411588966846, 0.013927980326116085, 0.029895801097154617, 0.02008231170475483, 0.03804440051317215, 0.06883092224597931, 0.0815289318561554, 0.0383402481675148, 0.15160253643989563, 0.48332712054252625, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0014211690286174417, 0.00040690117748454213, 0.0002714421134442091, 0.0014869242440909147, 0.0006475155241787434, 0.0002567152841947973, 0.00027297638007439673, 0.0005059852264821529, 0.00017537492385599762, 0.0012059591244906187, 0.0005902894772589207, 0.00043073901906609535, 0.00036999606527388096, 0.0013322837185114622, 0.000759427435696125, 0.002518307650461793, 0.003599978983402252, 0.004756716545671225, 0.011892932467162609, 0.0034102171193808317, 0.013851108960807323, 0.05907328426837921, 0.8907637596130371, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00505886459723115, 0.0008823543903417885, 0.0015358274104073644, 0.0016909865662455559, 0.0013438736787065864, 0.0004949708236381412, 0.0006572254933416843, 0.000552159093786031, 0.0005812101298943162, 0.002054560463875532, 0.0008380875224247575, 0.001006747712381184, 0.0011300853220745921, 0.0016139923827722669, 0.005022874567657709, 0.014576874673366547, 0.0037143337540328503, 0.004681476857513189, 0.007913822308182716, 0.006778388284146786, 0.012291365303099155, 0.027733024209737778, 0.10214646905660629, 0.7957004308700562, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00400469871237874, 0.0004576975479722023, 0.0015645211096853018, 0.0031372052617371082, 0.001993746031075716, 0.00037892640102654696, 0.00049577810568735, 0.000569801835808903, 0.001005206722766161, 0.0023432248272001743, 0.0005772155127488077, 0.0016756145050749183, 0.003078736364841461, 0.0012285250704735518, 0.005278328433632851, 0.00632805610075593, 0.003151731565594673, 0.0053193108178675175, 0.0213831327855587, 0.006147712003439665, 0.012332223355770111, 0.07402164489030838, 0.1211402490735054, 0.561111569404602, 0.16127507388591766, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0011286542285233736, 4.9889109504874796e-05, 4.129114677198231e-05, 9.24961204873398e-05, 5.513622454600409e-05, 3.8617254176642746e-05, 5.5920278100529686e-05, 0.00014896267384756356, 5.509220500243828e-06, 6.817427492933348e-05, 3.6204455682309344e-05, 4.623961285687983e-05, 0.001065501943230629, 5.91987554798834e-05, 7.787253707647324e-05, 0.00038421223871409893, 0.00011692437692545354, 0.0002242824120912701, 0.000508398690726608, 0.0001562813704367727, 0.00032331488910131156, 0.0007086826371960342, 0.0009207415860146284, 0.0008559715934097767, 0.004414450377225876, 0.9884170889854431, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0024299705401062965, 0.003659637412056327, 0.0006344981375150383, 0.0006202656077221036, 0.00298767676576972, 0.00034763681469485164, 0.0005059166578575969, 0.00041149763274006546, 0.00021321582607924938, 0.0006509345257654786, 0.0003955696302000433, 0.16986453533172607, 0.002188899554312229, 0.0006812380161136389, 0.000868376693688333, 0.008905136026442051, 0.0014638527063652873, 0.0013394575798884034, 0.006495129782706499, 0.007638717070221901, 0.004619399085640907, 0.003161790780723095, 0.014200896956026554, 0.046396613121032715, 0.02278606779873371, 0.21939247846603394, 0.4771406054496765, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015610319562256336, 0.0009648068808019161, 0.0010281825670972466, 0.007148513570427895, 0.005316650029271841, 0.0005432680482044816, 0.0007839886820875108, 0.0004708298947662115, 0.0022980361245572567, 0.005463092587888241, 0.0005483070272020996, 0.0008661497267894447, 0.0009035550756379962, 0.0009251966839656234, 0.008375934325158596, 0.0014478107914328575, 0.0020614899694919586, 0.003693382255733013, 0.017008859664201736, 0.006138675846159458, 0.007041892036795616, 0.04128709062933922, 0.06922487169504166, 0.26617181301116943, 0.05024212971329689, 0.11445137113332748, 0.13764896988868713, 0.23233483731746674, 0.0, 0.0, 0.0, 0.0, 0.0], [0.021895311772823334, 0.0013514558086171746, 0.0013727976474910975, 0.005157721694558859, 0.01093280129134655, 0.00044522207463160157, 0.0016178301302716136, 0.0005713265272788703, 0.0007700274582020938, 0.003093535080552101, 0.0003616345056798309, 0.0009329011081717908, 0.001903240685351193, 0.0005254389252513647, 0.0007036188035272062, 0.0015780640533193946, 0.0010200472315773368, 0.0009346723672933877, 0.005167578347027302, 0.0007855422445572913, 0.0028796829283237457, 0.006337142549455166, 0.04951414093375206, 0.021886898204684258, 0.03004683181643486, 0.06941598653793335, 0.02995387092232704, 0.061116255819797516, 0.6677284240722656, 0.0, 0.0, 0.0, 0.0], [0.0002946656895801425, 8.08511977083981e-05, 0.00011992631334578618, 0.00017154013039544225, 0.0003899005532730371, 8.630252705188468e-05, 5.253727795206942e-05, 6.616376049350947e-05, 0.0001137837243732065, 6.200082134455442e-05, 0.00011238027218496427, 0.0005867315921932459, 9.785001748241484e-05, 0.00021334877237677574, 0.00040444283513352275, 0.0004726238548755646, 0.0004730736545752734, 0.0005199519800953567, 0.0008571160142309964, 0.00022720196284353733, 0.0015976768918335438, 0.0014780627097934484, 0.0016501408535987139, 0.009339644573628902, 0.01279268879443407, 0.012606951408088207, 0.021659579128026962, 0.02886745147407055, 0.07784783840179443, 0.8267576694488525, 0.0, 0.0, 0.0], [0.0006396545795723796, 0.00019920240447390825, 0.0003378460241947323, 0.00032286165514960885, 0.00038839809712953866, 2.2511459974339232e-05, 0.00011559655831661075, 1.968370270333253e-05, 8.749768312554806e-05, 0.00016675743972882628, 2.1353174815885723e-05, 0.00018257695774082094, 0.002879622159525752, 3.2708376238588244e-05, 6.355983350658789e-05, 0.00090785016072914, 6.31420552963391e-05, 4.9545076763024554e-05, 0.005010190419852734, 0.00021555765124503523, 0.00018923325114883482, 0.00023579830303788185, 0.0021191714331507683, 0.0034752420615404844, 0.0004957506898790598, 0.012673038989305496, 0.0026274591218680143, 0.0011926377192139626, 0.028011513873934746, 0.1421458125114441, 0.7951081991195679, 0.0, 0.0], [0.005492142401635647, 0.0014179605059325695, 6.310038588708267e-05, 0.0009282368700951338, 0.0004675872914958745, 9.701211820356548e-05, 5.678690649801865e-05, 6.824053707532585e-05, 4.054806777276099e-05, 0.00021011428907513618, 6.534699787152931e-05, 0.00011957778770010918, 9.082323231268674e-05, 9.077344293473288e-05, 0.00013827496150042862, 0.00041228361078538, 0.00016900571063160896, 0.00013570708688348532, 0.002428320934996009, 0.00020654806576203555, 0.0004959595971740782, 0.00016561975644435734, 0.001825400977395475, 0.001680815708823502, 0.001018735347315669, 0.01507147029042244, 0.003408371703699231, 0.003384194802492857, 0.01886550709605217, 0.010037437081336975, 0.008259555324912071, 0.923088550567627, 0.0], [0.009386110119521618, 0.0017380556091666222, 0.0018091682577505708, 0.0014350252458825707, 0.0016812816029414535, 0.001979130320250988, 0.0014143437147140503, 0.0021127830259501934, 0.0012016963446512818, 0.0010467646643519402, 0.001253248774446547, 0.0012125695357099175, 0.0006952421390451491, 0.0016850176034495234, 0.001290497719310224, 0.0014949428150430322, 0.003052463522180915, 0.006176403723657131, 0.003993155900388956, 0.002994521288201213, 0.008827597834169865, 0.009542387910187244, 0.010477354750037193, 0.0214514322578907, 0.062213558703660965, 0.026861043646931648, 0.033906422555446625, 0.11987609416246414, 0.05027468875050545, 0.026796849444508553, 0.031165700405836105, 0.10728635638952255, 0.4436681270599365]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10520488768815994, 0.8947951793670654, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.03907204046845436, 0.0020172216463834047, 0.9589106440544128, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015578641556203365, 0.0008392541203647852, 0.0006979768513701856, 0.9828841090202332, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.008856471627950668, 9.934287845680956e-06, 1.1174832252436318e-05, 0.000302469969028607, 0.9908198714256287, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.35445019602775574, 0.030589278787374496, 0.059889569878578186, 0.022903528064489365, 0.0474759079515934, 0.4846915602684021, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04476895555853844, 0.001566466293297708, 0.00037738497485406697, 0.0002507324970792979, 0.00040889676893129945, 0.0002606561465654522, 0.9523669481277466, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09058346599340439, 0.0011470747413113713, 0.006027346942573786, 0.000546847702935338, 0.0017094232607632875, 0.00378508516587317, 0.002684543142095208, 0.8935161828994751, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0060107228346168995, 2.4289069187943824e-05, 0.00031266745645552874, 1.8682934751268476e-05, 0.0002979248820338398, 9.904525541060138e-06, 6.619561190746026e-06, 7.912206569926639e-07, 0.9933184385299683, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.004546814598143101, 0.0002001934335567057, 0.00029920213273726404, 0.001336393179371953, 0.0003267655265517533, 8.741358215047512e-07, 1.7415688489563763e-05, 2.783414174700738e-07, 0.00029665243346244097, 0.9929754734039307, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12262558937072754, 0.02392657659947872, 0.03827648609876633, 0.016171958297491074, 0.030366230756044388, 0.29375940561294556, 0.10661352425813675, 0.0705994963645935, 0.04190007597208023, 0.017833102494478226, 0.23792758584022522, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0008121057762764394, 0.00011546660243766382, 0.00013500906061381102, 5.5392923968611285e-05, 0.0001317415590165183, 1.0027196140072192e-06, 1.3799102589473478e-06, 1.6001629887796298e-07, 1.8972989437315846e-06, 5.419660737970844e-07, 4.753100597554294e-07, 0.9987448453903198, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0013333020033314824, 0.00010533027671044692, 0.0007669601473025978, 0.0009021890582516789, 3.5708292216440896e-06, 8.120475740724942e-07, 1.4038076301403635e-07, 9.5612755046659e-08, 1.6865038787727826e-06, 2.9789011023240164e-05, 4.3813363959088747e-07, 1.1965776138822548e-06, 0.9968544840812683, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08159798383712769, 0.021388858556747437, 0.028877155855298042, 0.012675793841481209, 0.02493158169090748, 0.229964941740036, 0.10401230305433273, 0.06472097337245941, 0.03460092470049858, 0.015198714099824429, 0.18564164638519287, 0.020528046414256096, 0.005003852304071188, 0.17085722088813782, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0013488862896338105, 8.92121170181781e-05, 0.000685996375977993, 0.0006963219493627548, 2.1248622942948714e-05, 3.7633941474268795e-07, 6.2267383782455e-07, 4.3072165567537013e-07, 7.091880888765445e-06, 0.0013103694655001163, 1.611375211041377e-07, 9.54876213654643e-07, 0.00016453623538836837, 1.15319252813606e-07, 0.9956737160682678, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0010867505334317684, 0.000407249026466161, 0.00010700321581680328, 0.000678271462675184, 0.00011028484004782513, 1.318426257057581e-07, 2.660271150034532e-07, 4.6195768277357274e-07, 8.409714610024821e-06, 0.0007784898625686765, 5.109449574547398e-08, 8.792079029262823e-07, 4.934294702252373e-05, 3.4893233191723994e-08, 0.000114411988761276, 0.9966580867767334, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05868088826537132, 0.0187857486307621, 0.024067312479019165, 0.011013144627213478, 0.021456381306052208, 0.1911056488752365, 0.09902721643447876, 0.06065867841243744, 0.03016272373497486, 0.014801910147070885, 0.15370330214500427, 0.016615159809589386, 0.004385382868349552, 0.14085331559181213, 0.015014998614788055, 0.0053902058862149715, 0.13427793979644775, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07323037832975388, 0.02837456949055195, 0.02105824276804924, 0.007611881475895643, 0.01005358062684536, 0.06553948670625687, 0.14559759199619293, 0.08620844036340714, 0.01024126447737217, 0.007425738964229822, 0.045262690633535385, 0.006621456239372492, 0.001162177650257945, 0.03923853859305382, 0.008256880566477776, 0.0030656892340630293, 0.036122631281614304, 0.40492871403694153, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [7.94449370005168e-05, 1.8382123698756914e-06, 3.7134324060161816e-08, 1.5575307770632207e-05, 3.4723757380561437e-06, 1.3165096790501707e-09, 2.703870904952055e-08, 1.7186652101486288e-08, 9.2322629541286e-08, 1.350637921859743e-05, 4.4787887287789374e-10, 2.041673052843862e-09, 5.5420287026208825e-08, 3.0638666603799436e-10, 4.217719435928302e-07, 7.065880708978511e-06, 2.424439260462208e-10, 1.4397358860662735e-09, 0.9998784065246582, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.002934493590146303, 0.0005883841076865792, 0.004250307101756334, 0.0040284618735313416, 0.00011592944065341726, 4.3316217670508195e-06, 6.773899713152787e-06, 7.076467591105029e-05, 0.0002495265216566622, 0.0003159338375553489, 2.08243977795064e-06, 4.4092124880990013e-05, 0.00030446931486949325, 1.4702561657031765e-06, 0.0001976135972654447, 0.0013599261874333024, 1.2478651569836074e-06, 6.875188319099834e-06, 0.00019261146371718496, 0.9853246808052063, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04056532680988312, 0.017898326739668846, 0.020600972697138786, 0.009821100160479546, 0.01788281463086605, 0.15653997659683228, 0.09208228439092636, 0.05628826469182968, 0.026661338284611702, 0.014471272937953472, 0.12537629902362823, 0.013986770063638687, 0.00392792746424675, 0.11453618854284286, 0.013395379297435284, 0.004846664611250162, 0.10880246758460999, 0.049533966928720474, 0.005186409689486027, 0.004420871380716562, 0.10317535698413849, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02453630231320858, 0.0031243355479091406, 0.0009699289221316576, 4.6315435611177236e-05, 0.0001343475450994447, 0.0006453939131461084, 0.011668838560581207, 0.00032713444670662284, 0.0004573040350805968, 2.3976079319254495e-05, 0.0003445417678449303, 4.996419738745317e-05, 2.5757384719327092e-05, 0.0002821775560732931, 6.525318895000964e-06, 4.981805432180408e-06, 0.00023554006475023925, 0.0002664509229362011, 2.531113386794459e-05, 2.9761044061160646e-05, 0.00018809252651408315, 0.9566071033477783, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.002076001837849617, 0.00014591813669539988, 5.28989803569857e-05, 4.218077810946852e-05, 4.7387235099449754e-05, 4.14233511492057e-07, 0.00012964299821760505, 2.4251021386589855e-05, 1.8034043023362756e-06, 1.4839295090496307e-06, 1.4490007060885546e-07, 4.199936029181117e-06, 4.1068410894240515e-08, 1.0260500005188078e-07, 8.285463763968437e-07, 5.781907930213492e-07, 7.69515864362802e-08, 6.669622365507166e-08, 6.47262713755481e-05, 1.5700916264904663e-06, 5.6418304694716426e-08, 4.403035745781381e-06, 0.997401237487793, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0005159107386134565, 0.00011814631579909474, 3.2705611374694854e-05, 0.00021188720711506903, 4.563520269584842e-05, 2.924884086041857e-07, 3.0579394660890102e-06, 7.607361141026558e-08, 0.0001375367137370631, 0.0005283543141558766, 1.1576013747571778e-07, 4.028577222925378e-06, 2.079294745271909e-06, 8.805408668877135e-08, 0.00010051687422674149, 4.43161916336976e-05, 6.963698950812613e-08, 9.373040654736542e-08, 3.337768794153817e-05, 1.5888888810877688e-05, 5.6106753021367695e-08, 7.05264184830412e-08, 9.389270417159423e-06, 0.9981963038444519, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.06915225088596344, 0.014032105915248394, 0.005599097348749638, 0.0012576853623613715, 0.0008468001033179462, 0.009601173922419548, 0.008965768851339817, 0.052476901561021805, 0.00025028391974046826, 0.0009063858306035399, 0.005419905763119459, 0.000730921165086329, 0.0001918794005177915, 0.004396580625325441, 0.005964720156043768, 0.00035585410660132766, 0.0038770169485360384, 0.009727226570248604, 0.0011023489059880376, 0.0003644288226496428, 0.0034139587078243494, 0.0030523554887622595, 0.005698245484381914, 0.00029557300149463117, 0.7923205494880676, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0010461547644808888, 1.9557141058612615e-05, 1.7358959212288028e-06, 3.1030376703711227e-05, 2.73079531325493e-05, 1.0180698062356441e-08, 4.815767624677392e-07, 1.749424853869641e-07, 7.31240987761339e-08, 7.983068712746899e-07, 3.4643101720632785e-09, 4.9067175496020354e-06, 1.0718932230702194e-07, 2.352543715389288e-09, 2.3834076046114205e-07, 1.3731333865507622e-06, 1.828867168818249e-09, 6.247609984910696e-09, 9.208339179167524e-06, 1.6384256014134735e-06, 1.2523899695082719e-09, 8.271274221094416e-10, 8.456770046905149e-06, 3.118676431768108e-06, 9.241420961814129e-09, 0.9988435506820679, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0010544945253059268, 0.00021373217168729752, 0.00028157219639979303, 0.0011373378802090883, 0.00014783968799747527, 2.00685326490202e-06, 4.831355909118429e-06, 3.48233857039304e-06, 5.100344878883334e-06, 0.000124204860185273, 9.301529075855797e-07, 1.0582622962829191e-05, 1.1688286576827522e-05, 7.077518944242911e-07, 0.003411600599065423, 0.00019905276712961495, 5.995618153065152e-07, 4.643046622732072e-07, 1.9585138943511993e-05, 0.00011087791790487245, 4.7500583377768635e-07, 3.2808881655910227e-07, 4.535358584689675e-06, 5.1617142162285745e-05, 3.247476740853017e-07, 1.3071385183138773e-05, 0.9931889772415161, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02420075610280037, 0.002820128807798028, 0.004319296684116125, 0.0009108306840062141, 0.0017753936117514968, 0.003809914458543062, 0.0029533228371292353, 0.01671477034687996, 0.001393173704855144, 0.005003183614462614, 0.002276088809594512, 0.0005876408540643752, 0.0009901623707264662, 0.0018066932680085301, 0.0031312420032918453, 0.00023389369016513228, 0.0016082713846117258, 0.00991030503064394, 0.002853952581062913, 0.00020495128410402685, 0.0013712650397792459, 0.00691900635138154, 0.0010336573468521237, 0.00014528384781442583, 0.017489798367023468, 0.00046553125139325857, 0.00030240853084251285, 0.8847692012786865, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0005256092990748584, 3.528650950102019e-06, 1.703768475636025e-06, 9.408179903402925e-05, 0.6647976040840149, 1.6531491553450905e-07, 9.511112466498162e-07, 8.886883620107255e-07, 1.701671317277942e-05, 3.7421868910314515e-05, 6.832266308265389e-08, 9.810898518480826e-06, 7.548737102069936e-08, 4.9632518539510784e-08, 2.0393888462422183e-06, 7.686546268814709e-06, 4.066851388984105e-08, 2.6744315206883584e-08, 3.343187927384861e-05, 4.341078749803273e-07, 3.007988169656528e-08, 3.206056931048806e-08, 4.900574822386261e-06, 3.0609394343628082e-06, 1.4603195985785078e-08, 2.2145539332996123e-05, 9.21115315577481e-06, 3.164272044386962e-08, 0.33442792296409607, 0.0, 0.0, 0.0, 0.0], [0.004677004646509886, 0.0002802134840749204, 8.346093818545341e-05, 3.157015271426644e-06, 4.065350367454812e-07, 1.334351367177078e-07, 2.6423560484545305e-05, 1.14399881567806e-07, 2.787234905099467e-07, 3.7496099594136467e-06, 4.652897445112103e-08, 1.7213982346220291e-06, 1.8165600579322927e-07, 3.1484699292150253e-08, 1.8334249318741058e-07, 5.2147463236451586e-08, 2.5215808108214333e-08, 3.078440968806717e-08, 7.142029545548212e-08, 1.6640171907056356e-08, 1.7768222448921733e-08, 8.634525272555038e-08, 2.80096719507128e-06, 8.454135809188301e-07, 3.098357126418705e-08, 2.927866091795295e-07, 5.67622805647261e-07, 5.0554116448608966e-09, 2.9141034474378102e-08, 0.9949179887771606, 0.0, 0.0, 0.0], [5.636792411678471e-05, 0.00020023711840622127, 1.6319079350068932e-06, 1.5089327689565835e-06, 1.7779733752831817e-05, 5.95644145118257e-10, 6.940369075891795e-06, 5.034019867622419e-09, 2.1682852491267113e-07, 1.7164256860269234e-06, 1.9326265587871205e-10, 1.8911425314627195e-08, 9.538840117784275e-08, 1.2946883842790413e-10, 8.07715050399338e-09, 4.386615546536632e-05, 9.768683778554887e-11, 5.256080015669795e-10, 6.93216934450902e-05, 7.505880716962565e-08, 7.069290985928234e-11, 3.988046870517792e-08, 1.0610276149236597e-05, 2.2120707399153616e-06, 8.688633701403603e-10, 9.928487088473048e-06, 5.390084378831261e-09, 1.1354458345769203e-10, 3.060037442992325e-06, 3.8945552205404965e-07, 0.9995738863945007, 0.0, 0.0], [0.00035569758620113134, 1.9489458281896077e-05, 9.160975423583295e-06, 1.3085252248856705e-05, 2.3808875994291157e-05, 2.97190183573548e-08, 4.1605944716138765e-05, 1.7756508441379992e-07, 6.896241870890663e-07, 1.050624064191652e-06, 1.2230765733534099e-08, 4.290548076824052e-06, 3.295830310889869e-07, 9.21108966878137e-09, 1.7466413737565745e-06, 4.68278949483647e-06, 7.4427397578347154e-09, 9.673819079125678e-09, 3.9665097574470565e-05, 7.324898660954204e-07, 5.5436903956262995e-09, 1.0396706073834139e-07, 1.4401575754163787e-05, 8.873340448189992e-06, 5.98941607421466e-09, 1.5627783795935102e-05, 2.9811219519615406e-06, 6.874342872720263e-09, 4.854650796914939e-06, 7.0722649070376065e-06, 6.210685569385532e-06, 0.999423623085022, 0.0], [0.05455271154642105, 0.01642906479537487, 0.01600833795964718, 0.006986881140619516, 0.02488415502011776, 0.07376932352781296, 0.07933780550956726, 0.05594552680850029, 0.02726820856332779, 0.013368779793381691, 0.05590308830142021, 0.007223027292639017, 0.005426580552011728, 0.05012328550219536, 0.00827906746417284, 0.0030609637033194304, 0.046822965145111084, 0.04751850664615631, 0.0028449445962905884, 0.007037974428385496, 0.042827170342206955, 0.0513819120824337, 0.02728237397968769, 0.026375338435173035, 0.026159211993217468, 0.011997989378869534, 0.006220121867954731, 0.03648780658841133, 0.011124704033136368, 0.005218563135713339, 0.0025592022575438023, 0.0026040268130600452, 0.14697040617465973]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9192656874656677, 0.08073433488607407, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.45437392592430115, 0.4265042543411255, 0.11912186443805695, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5267521739006042, 0.22321631014347076, 0.12420626729726791, 0.12582528591156006, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3903489112854004, 0.17099407315254211, 0.06872710585594177, 0.1837182343006134, 0.1862117052078247, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19744226336479187, 0.25436851382255554, 0.14488449692726135, 0.18925853073596954, 0.20821937918663025, 0.00582678010687232, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19473573565483093, 0.13241828978061676, 0.14391927421092987, 0.15120737254619598, 0.3002447485923767, 0.02058463543653488, 0.056889984756708145, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1269666850566864, 0.21597377955913544, 0.16089318692684174, 0.1969882994890213, 0.1884462684392929, 0.013544929213821888, 0.07520709931850433, 0.021979698911309242, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2912123203277588, 0.05954226851463318, 0.11079408973455429, 0.09717857837677002, 0.22345606982707977, 0.016262585297226906, 0.08719441294670105, 0.024762287735939026, 0.08959739655256271, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.22981953620910645, 0.038026757538318634, 0.1601383537054062, 0.11780407279729843, 0.1618119478225708, 0.03552818298339844, 0.07561112195253372, 0.020756039768457413, 0.10670627653598785, 0.05379769951105118, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13314427435398102, 0.18858656287193298, 0.10808095335960388, 0.15994971990585327, 0.1773921251296997, 0.003598014824092388, 0.0250620748847723, 0.01286292728036642, 0.0745309367775917, 0.11275240778923035, 0.004039982799440622, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1674640029668808, 0.1435522586107254, 0.06118571758270264, 0.17358075082302094, 0.10113830864429474, 0.01755347289144993, 0.03239530324935913, 0.007995500229299068, 0.05453447997570038, 0.07883201539516449, 0.017258215695619583, 0.14450997114181519, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.26996830105781555, 0.13029062747955322, 0.06605685502290726, 0.061561424285173416, 0.08437130600214005, 0.01984662190079689, 0.01631278730928898, 0.012965923175215721, 0.03231464698910713, 0.05254792049527168, 0.01986861415207386, 0.13695642352104187, 0.09693864732980728, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09820353239774704, 0.13618646562099457, 0.0800577849149704, 0.12152864784002304, 0.1357937604188919, 0.0024323570542037487, 0.01778949797153473, 0.009086488746106625, 0.05637527257204056, 0.08521638810634613, 0.0027533371467143297, 0.1520378440618515, 0.0994671955704689, 0.00307140382938087, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1701289564371109, 0.08885712921619415, 0.09089689701795578, 0.08085790276527405, 0.11427336931228638, 0.010901479050517082, 0.014788412488996983, 0.005497370846569538, 0.051512569189071655, 0.05330684036016464, 0.010595815256237984, 0.09442280977964401, 0.13098116219043732, 0.011153682135045528, 0.07182558625936508, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10365856438875198, 0.03588540852069855, 0.08803567290306091, 0.06087937951087952, 0.08348330855369568, 0.03080574795603752, 0.029615936800837517, 0.010918705724179745, 0.04049992188811302, 0.04288605973124504, 0.03219296410679817, 0.15008129179477692, 0.12658776342868805, 0.0349021814763546, 0.05825432389974594, 0.07131282985210419, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07047097384929657, 0.0960099846124649, 0.056215789169073105, 0.08699887990951538, 0.09934122860431671, 0.0016298787668347359, 0.01240861602127552, 0.006380431819707155, 0.041014038026332855, 0.06348712742328644, 0.0018482195446267724, 0.11387647688388824, 0.07328979671001434, 0.0020712940022349358, 0.07246166467666626, 0.20012779533863068, 0.002367778215557337, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05659440532326698, 0.08906208723783493, 0.04703241214156151, 0.09457876533269882, 0.08793889731168747, 0.001972166122868657, 0.017767680808901787, 0.005990432109683752, 0.06014860421419144, 0.08834964036941528, 0.0022637685760855675, 0.1046152412891388, 0.06079366058111191, 0.00255662202835083, 0.07818624377250671, 0.19496025145053864, 0.0029299911111593246, 0.004259058274328709, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1597062051296234, 0.026096442714333534, 0.04515180364251137, 0.0349758081138134, 0.08030673116445541, 0.05469036474823952, 0.04078545421361923, 0.02651304565370083, 0.02725193277001381, 0.03919816389679909, 0.05351434648036957, 0.0484292209148407, 0.0626426488161087, 0.05558208376169205, 0.05559664964675903, 0.04792959615588188, 0.05676326900720596, 0.04892301186919212, 0.035943206399679184, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08663122355937958, 0.09557436406612396, 0.036738116294145584, 0.06946705281734467, 0.05681459605693817, 0.00819021463394165, 0.022195832803845406, 0.0076428246684372425, 0.03146827220916748, 0.07555203884840012, 0.0089652631431818, 0.05263480916619301, 0.04491106793284416, 0.009935053065419197, 0.04481377825140953, 0.15198466181755066, 0.0109334047883749, 0.007993670180439949, 0.16402564942836761, 0.01352810300886631, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04783454164862633, 0.06669356673955917, 0.03798917308449745, 0.06040368974208832, 0.06899043172597885, 0.0010267288889735937, 0.008172674104571342, 0.004158170893788338, 0.028268268331885338, 0.044072382152080536, 0.0011586467735469341, 0.0783291831612587, 0.05082900822162628, 0.0013004952343180776, 0.049042947590351105, 0.14145004749298096, 0.0014934344217181206, 0.0025566834956407547, 0.17534081637859344, 0.12910208106040955, 0.0017870558658614755, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04114089906215668, 0.04766072705388069, 0.03586998209357262, 0.06666103005409241, 0.07969274371862411, 0.00236507598310709, 0.02034018747508526, 0.004854495171457529, 0.0643620491027832, 0.06737152487039566, 0.0027073686942458153, 0.05596053972840309, 0.05648668855428696, 0.0030767249409109354, 0.05354386568069458, 0.15165923535823822, 0.003539716824889183, 0.0046739280223846436, 0.16402718424797058, 0.05968156456947327, 0.004154939670115709, 0.010169521905481815, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0662880688905716, 0.04092540964484215, 0.02732268162071705, 0.04505367949604988, 0.06575217843055725, 0.015657562762498856, 0.02788187935948372, 0.013711260631680489, 0.03174294903874397, 0.05280746519565582, 0.015447776764631271, 0.032181113958358765, 0.032901059836149216, 0.01629886031150818, 0.0266366358846426, 0.09772849082946777, 0.017114203423261642, 0.01691945269703865, 0.09847065806388855, 0.04787908494472504, 0.018412545323371887, 0.015331599861383438, 0.17753532528877258, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09801430255174637, 0.026328807696700096, 0.03586815297603607, 0.05713058263063431, 0.06943874061107635, 0.012128958478569984, 0.020909087732434273, 0.008747960440814495, 0.02699965611100197, 0.037266768515110016, 0.011716341599822044, 0.08499539643526077, 0.05501333996653557, 0.012196713127195835, 0.03563909977674484, 0.08789661526679993, 0.012970106676220894, 0.010851015336811543, 0.06408385187387466, 0.059639204293489456, 0.01449217926710844, 0.006093734409660101, 0.09761437773704529, 0.053964968770742416, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02769325114786625, 0.04073771461844444, 0.031223542988300323, 0.058790355920791626, 0.056018248200416565, 0.0015714796027168632, 0.011908041313290596, 0.003339792136102915, 0.051765501499176025, 0.05291244387626648, 0.0018259818898513913, 0.060300350189208984, 0.028357185423374176, 0.0020878082141280174, 0.05086478963494301, 0.07898838073015213, 0.0023987055756151676, 0.002904822351410985, 0.11135734617710114, 0.04586018621921539, 0.0028849171940237284, 0.011431250721216202, 0.1234055683016777, 0.13488474488258362, 0.006487556733191013, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15390776097774506, 0.023786796256899834, 0.05320245772600174, 0.03963885083794594, 0.05711062252521515, 0.015628937631845474, 0.02020607329905033, 0.009252368472516537, 0.02131185494363308, 0.02902994491159916, 0.014741902239620686, 0.0612383633852005, 0.10112591087818146, 0.015157817862927914, 0.04335402697324753, 0.04090719670057297, 0.01573265716433525, 0.020073410123586655, 0.053749360144138336, 0.03468616306781769, 0.01698937825858593, 0.006368495989590883, 0.04042017087340355, 0.031060419976711273, 0.013937168754637241, 0.06738192588090897, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08555091917514801, 0.035057418048381805, 0.05885257571935654, 0.03497908264398575, 0.05178743600845337, 0.0025119921192526817, 0.0041512190364301205, 0.0020859173964709044, 0.014081120491027832, 0.02449135296046734, 0.0023760052863508463, 0.09535116702318192, 0.07766135036945343, 0.0024908811319619417, 0.045726776123046875, 0.05774116516113281, 0.002660380443558097, 0.0033477561082690954, 0.047998279333114624, 0.02678881771862507, 0.0029587154276669025, 0.004655787255614996, 0.03607099875807762, 0.021599192172288895, 0.004788633435964584, 0.22052206099033356, 0.03371305391192436, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.020706716924905777, 0.037194062024354935, 0.01972370594739914, 0.038942236453294754, 0.03182699903845787, 0.0012253678869456053, 0.008815747685730457, 0.002405293518677354, 0.0345795564353466, 0.047298070043325424, 0.001395374652929604, 0.05148206278681755, 0.02048674412071705, 0.0015608376124873757, 0.03826117143034935, 0.04863753169775009, 0.0017825308023020625, 0.0025446040090173483, 0.06541350483894348, 0.0418856181204319, 0.0020960604306310415, 0.009401390329003334, 0.08460027724504471, 0.09019991755485535, 0.0054032751359045506, 0.22932568192481995, 0.05692270025610924, 0.005882916506379843, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08582809567451477, 0.039671752601861954, 0.014841826632618904, 0.056536220014095306, 0.05960613489151001, 0.0069875167682766914, 0.010110098868608475, 0.002887725830078125, 0.015322853811085224, 0.02592875435948372, 0.006830447353422642, 0.033548954874277115, 0.021093523129820824, 0.007213158532977104, 0.019323380663990974, 0.05298228561878204, 0.007658614311367273, 0.005157842300832272, 0.06523183733224869, 0.018170006573200226, 0.008534705266356468, 0.0043681650422513485, 0.026912428438663483, 0.049627941101789474, 0.004404331557452679, 0.19388671219348907, 0.03933224827051163, 0.005363550037145615, 0.11263895034790039, 0.0, 0.0, 0.0, 0.0], [0.07040347903966904, 0.05926162749528885, 0.015334847383201122, 0.0241390373557806, 0.015728840604424477, 0.019492102786898613, 0.017311513423919678, 0.017303066328167915, 0.020563244819641113, 0.0451558418571949, 0.021815825253725052, 0.07079727947711945, 0.030063964426517487, 0.02354060858488083, 0.020684480667114258, 0.02116367779672146, 0.02644512802362442, 0.023540044203400612, 0.05663726478815079, 0.0336947925388813, 0.030707288533449173, 0.015993360430002213, 0.015474079176783562, 0.030803624540567398, 0.03180558606982231, 0.07969609647989273, 0.03437487408518791, 0.034537144005298615, 0.030624371021986008, 0.06290692836046219, 0.0, 0.0, 0.0], [0.05338500067591667, 0.029621528461575508, 0.021030094474554062, 0.023695707321166992, 0.033436402678489685, 0.01839382015168667, 0.024060707539319992, 0.02281002327799797, 0.013972364366054535, 0.028813544660806656, 0.019059743732213974, 0.015371965244412422, 0.02533530630171299, 0.019971953704953194, 0.014075737446546555, 0.0388445146381855, 0.021346209570765495, 0.019321434199810028, 0.056326594203710556, 0.021360008046030998, 0.023394575342535973, 0.023136639967560768, 0.06416381895542145, 0.06724688410758972, 0.0346950888633728, 0.043490175157785416, 0.03032725676894188, 0.03285712003707886, 0.053275078535079956, 0.0659981295466423, 0.041182488203048706, 0.0, 0.0], [0.18177089095115662, 0.019533155485987663, 0.028002941980957985, 0.02911018393933773, 0.04134833440184593, 0.01801948994398117, 0.010471724905073643, 0.008382909931242466, 0.01533578522503376, 0.021839451044797897, 0.01689334586262703, 0.03806138411164284, 0.03643865883350372, 0.017464255914092064, 0.026129068806767464, 0.04011504724621773, 0.018317215144634247, 0.012488218024373055, 0.05136970803141594, 0.01944384165108204, 0.019889770075678825, 0.0035764521453529596, 0.026232963427901268, 0.018429189920425415, 0.011723755858838558, 0.0327298603951931, 0.034808199852705, 0.01203504391014576, 0.06030292063951492, 0.00787330698221922, 0.042272936552762985, 0.07959000021219254, 0.0], [0.015003404580056667, 0.02033485472202301, 0.007813435047864914, 0.016553575173020363, 0.01646609604358673, 0.00023809655976947397, 0.0019598929211497307, 0.0006898260326124728, 0.006483218166977167, 0.010734074749052525, 0.0002443529956508428, 0.014634673483669758, 0.00924762524664402, 0.0002605919726192951, 0.009339917451143265, 0.042726580053567886, 0.0002886394504457712, 0.0003811670758295804, 0.054749373346567154, 0.09452768415212631, 0.00034211305319331586, 0.001727019902318716, 0.04745933413505554, 0.02661379985511303, 0.0010238662362098694, 0.1653798222541809, 0.014187106862664223, 0.0013092352310195565, 0.04770654812455177, 0.0111984983086586, 0.2205076366662979, 0.1394261121749878, 0.0004418626776896417]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9333657622337341, 0.06663428246974945, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.32452359795570374, 0.5923718214035034, 0.08310455828905106, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1415974646806717, 0.514380931854248, 0.28144142031669617, 0.06258013844490051, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.25262272357940674, 0.058340705931186676, 0.10458870232105255, 0.4411159157752991, 0.14333190023899078, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.18051563203334808, 0.09828907996416092, 0.10873956233263016, 0.12262694537639618, 0.09995546191930771, 0.38987332582473755, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1719229817390442, 0.023320995271205902, 0.12332706153392792, 0.06076495721936226, 0.05971711874008179, 0.347591757774353, 0.21335510909557343, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09627032279968262, 0.04683619737625122, 0.028257304802536964, 0.03239249438047409, 0.0586504191160202, 0.15829356014728546, 0.3099648952484131, 0.2693347930908203, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09948235750198364, 0.0646418035030365, 0.028671829029917717, 0.0730566680431366, 0.02978716976940632, 0.1396002322435379, 0.20754705369472504, 0.265802264213562, 0.09141054004430771, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05632037669420242, 0.029885854572057724, 0.01341991126537323, 0.0024075533729046583, 0.013245469890534878, 0.045645419508218765, 0.05522569268941879, 0.10027101635932922, 0.6579986810684204, 0.02558007836341858, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04266994819045067, 0.010426479391753674, 0.010652213357388973, 0.013446621596813202, 0.00935314130038023, 0.04015978425741196, 0.0710517093539238, 0.1373765617609024, 0.10423263162374496, 0.21736638247966766, 0.34326446056365967, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0296839140355587, 0.002561642089858651, 0.004407483618706465, 0.034248705953359604, 0.014238743111491203, 0.028001822531223297, 0.02228802628815174, 0.07105964422225952, 0.04026233032345772, 0.5543041825294495, 0.17333054542541504, 0.025612974539399147, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04039592295885086, 0.008631582371890545, 0.005494222976267338, 0.003864138852804899, 0.023363104090094566, 0.027207026258111, 0.025677336379885674, 0.050485219806432724, 0.0510561428964138, 0.05041782185435295, 0.14164337515830994, 0.511528491973877, 0.060235634446144104, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.028706390410661697, 0.004808460362255573, 0.004723656922578812, 0.005382601171731949, 0.003456039587035775, 0.014470163732767105, 0.024037377908825874, 0.041626207530498505, 0.03567296266555786, 0.07060738652944565, 0.10830992460250854, 0.05811614543199539, 0.2062121331691742, 0.3938705325126648, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05342644825577736, 0.011923279613256454, 0.015572259202599525, 0.0021123222541064024, 0.005851038731634617, 0.02217092178761959, 0.03818469122052193, 0.02741156704723835, 0.054642871022224426, 0.04910830780863762, 0.12328378856182098, 0.05287173017859459, 0.07545649260282516, 0.3701530694961548, 0.09783118963241577, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.008346711285412312, 0.00027852741186507046, 0.0011785993119701743, 0.0021395485382527113, 0.002476169029250741, 0.002918292535468936, 0.004576893523335457, 0.0039655971340835094, 0.004973022732883692, 0.011347771622240543, 0.013014075346291065, 0.005545974709093571, 0.02089560590684414, 0.03838539123535156, 0.8750316500663757, 0.004926194902509451, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02170032449066639, 0.002678664866834879, 0.0023839243222028017, 0.002815627260133624, 0.001556233735755086, 0.006139412056654692, 0.009039806202054024, 0.01374087855219841, 0.012412214651703835, 0.02468654327094555, 0.03406905382871628, 0.017561480402946472, 0.06082957610487938, 0.12004052847623825, 0.11516803503036499, 0.12663689255714417, 0.42854079604148865, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01968139037489891, 0.0028661887627094984, 0.002752861240878701, 0.0023642012383788824, 0.0022205759305506945, 0.0047769746743142605, 0.005015391390770674, 0.007020775228738785, 0.006603003013879061, 0.01582488976418972, 0.02177913300693035, 0.02237590402364731, 0.03905554488301277, 0.07342638820409775, 0.07227564603090286, 0.0810362920165062, 0.2556319534778595, 0.36529290676116943, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.034438684582710266, 0.0024295076727867126, 0.0027038780972361565, 0.0010108931455761194, 0.0028212429024279118, 0.009310661815106869, 0.00747407041490078, 0.010526392608880997, 0.01934259757399559, 0.006583388429135084, 0.030525274574756622, 0.008166542276740074, 0.023945646360516548, 0.08047989010810852, 0.04428938403725624, 0.048865143209695816, 0.23266319930553436, 0.38709720969200134, 0.04732644557952881, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0036976831033825874, 0.006450952962040901, 0.0004921727231703699, 0.0003069589729420841, 0.00012400773994158953, 0.0010727370390668511, 0.0008858796209096909, 0.001380636473186314, 0.0008200175943784416, 0.0002731183485593647, 0.002783065428957343, 0.03529426082968712, 0.0035613791551440954, 0.007148314733058214, 0.001647408353164792, 0.016731660813093185, 0.01917281188070774, 0.022339196875691414, 0.8733668327331543, 0.002450960921123624, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.013628787361085415, 0.001438445644453168, 0.0012188641121611, 0.0012915851548314095, 0.0006118023884482682, 0.0021400381810963154, 0.0027469571214169264, 0.0033360477536916733, 0.003091872902587056, 0.005988912656903267, 0.007237219717353582, 0.003547744592651725, 0.012891951948404312, 0.022229159250855446, 0.020824221894145012, 0.02629532851278782, 0.0764816403388977, 0.16261443495750427, 0.1263866275548935, 0.10034442692995071, 0.4056538939476013, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.021432969719171524, 0.0007291649235412478, 0.0010889691766351461, 0.0008559620473533869, 0.0010556603083387017, 0.002354251453652978, 0.0018633343279361725, 0.002486992860212922, 0.004011375363916159, 0.0036456233356148005, 0.006636620499193668, 0.0024591651745140553, 0.00810664240270853, 0.01950138807296753, 0.013945515267550945, 0.01683160476386547, 0.06727856397628784, 0.1020374447107315, 0.07506179064512253, 0.03785506263375282, 0.3474159240722656, 0.2633460462093353, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01837441883981228, 0.0016340153524652123, 0.0002648660447448492, 0.0006277099018916488, 0.0007212890777736902, 0.0022947092074900866, 0.0014375777682289481, 0.0024335708003491163, 0.0010225051082670689, 0.002173094777390361, 0.005665970034897327, 0.000585706380661577, 0.004730311222374439, 0.01578962802886963, 0.009121065959334373, 0.006447889842092991, 0.049651872366666794, 0.07221098989248276, 0.04530426487326622, 0.02253621071577072, 0.23380489647388458, 0.28100958466529846, 0.22215785086154938, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01601027511060238, 0.00041907597915269434, 0.0016793111572042108, 0.0020331626292318106, 0.0011126026511192322, 0.0023041211534291506, 0.001641488284803927, 0.0020452297758311033, 0.0008826447883620858, 0.003531813621520996, 0.00466074887663126, 0.001146126538515091, 0.004025280009955168, 0.011729470454156399, 0.006906012073159218, 0.005978981498628855, 0.03413612022995949, 0.053919851779937744, 0.07536919414997101, 0.015544144436717033, 0.153072789311409, 0.21672524511814117, 0.23267339169979095, 0.1524529755115509, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01641550101339817, 0.0010881004855036736, 0.001302703283727169, 0.001233565155416727, 0.0010325239272788167, 0.0013605316635221243, 0.001135275000706315, 0.0009713207837194204, 0.0017580230487510562, 0.002354995347559452, 0.0022677984088659286, 0.0026170071214437485, 0.0032849612180143595, 0.00556444376707077, 0.0072877430357038975, 0.006582668051123619, 0.0169111005961895, 0.023348137736320496, 0.033363793045282364, 0.01206086203455925, 0.08410514146089554, 0.10776982456445694, 0.12741318345069885, 0.2500958740711212, 0.2886749505996704, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.031117741018533707, 0.0015063054161146283, 0.0010839324677363038, 0.002421524142846465, 0.001382686896249652, 0.0037937266752123833, 0.001633929554373026, 0.003119518980383873, 0.0012812635395675898, 0.002068523084744811, 0.0047601195983588696, 0.0008757320465520024, 0.008592123165726662, 0.0093536376953125, 0.011993243359029293, 0.005963001865893602, 0.022028174251317978, 0.03411327302455902, 0.009586759842932224, 0.013681814074516296, 0.08417054265737534, 0.08917812258005142, 0.10806837677955627, 0.15560784935951233, 0.37236863374710083, 0.020249491557478905, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.014838481321930885, 0.0005757318576797843, 0.004743653815239668, 0.00045677347225137055, 0.0007412288105115294, 0.0015000010607764125, 0.0008866073912940919, 0.0008452099282294512, 0.00010359263978898525, 0.000906514236703515, 0.0018096797866746783, 0.0602911040186882, 0.001537756179459393, 0.0033079395070672035, 0.0013713664375245571, 0.002082675462588668, 0.008643986657261848, 0.009088865481317043, 0.007760955020785332, 0.00580833712592721, 0.033865127712488174, 0.03987184166908264, 0.05374552309513092, 0.036947719752788544, 0.12259436398744583, 0.5718704462051392, 0.013804465532302856, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01857001520693302, 0.0012143177445977926, 0.001288044499233365, 0.0010612568585202098, 0.0007490046555176377, 0.001048562815412879, 0.000868787697982043, 0.0005973864463157952, 0.0012938270810991526, 0.001404251204803586, 0.0011055096983909607, 0.0005003446131013334, 0.0020262624602764845, 0.0021463141310960054, 0.0028737925458699465, 0.0026413712184876204, 0.005802977364510298, 0.0081913648173213, 0.00833908375352621, 0.005543305538594723, 0.026558682322502136, 0.03308920934796333, 0.032507359981536865, 0.09199848771095276, 0.10410483181476593, 0.16861778497695923, 0.208954319357872, 0.2669035494327545, 0.0, 0.0, 0.0, 0.0, 0.0], [0.016654442995786667, 0.0007248665788210928, 0.0008980666170828044, 0.0031616878695786, 0.0005522824940271676, 0.0016607288271188736, 0.00074861227767542, 0.0009652797016315162, 0.0003842804580926895, 0.0022577813360840082, 0.0015545616624876857, 0.0007037006434984505, 0.0014688181690871716, 0.002799882087856531, 0.003013978712260723, 0.005369038321077824, 0.006552346982061863, 0.008701568469405174, 0.007138868793845177, 0.006516328547149897, 0.026185575872659683, 0.03929203003644943, 0.03155006840825081, 0.02434305101633072, 0.1067076250910759, 0.15159794688224792, 0.13674212992191315, 0.3201395273208618, 0.09161490201950073, 0.0, 0.0, 0.0, 0.0], [0.0372268483042717, 0.0007442747591994703, 0.0019180340459570289, 0.0014290640829131007, 0.0025597368367016315, 0.003292836481705308, 0.0009007275220938027, 0.001544511178508401, 0.0003557214804459363, 0.0004492733278311789, 0.0026688158977776766, 0.0014046661090105772, 0.0017281017499044538, 0.004542736802250147, 0.0012941033346578479, 0.004677668213844299, 0.009138188324868679, 0.008992945775389671, 0.005343662574887276, 0.0020318389870226383, 0.030930904671549797, 0.021215269342064857, 0.03639978542923927, 0.056794628500938416, 0.11195950210094452, 0.011458109132945538, 0.03353285789489746, 0.2658853828907013, 0.2469542920589447, 0.09262555837631226, 0.0, 0.0, 0.0], [0.044088784605264664, 0.0011082949349656701, 0.0033875906374305487, 0.004853191319853067, 0.002864320995286107, 0.004500862676650286, 0.0014518116367980838, 0.0021924942266196012, 0.0007902359939180315, 0.0006717380601912737, 0.003577157622203231, 0.0005131773068569601, 0.012861719354987144, 0.0051168459467589855, 0.0030142527539283037, 0.005887466017156839, 0.010453412309288979, 0.013818331062793732, 0.008691059425473213, 0.011229269206523895, 0.0314009003341198, 0.023181650787591934, 0.03044436313211918, 0.03520682081580162, 0.09999866038560867, 0.006090165581554174, 0.14714214205741882, 0.16393053531646729, 0.12749530375003815, 0.1935775727033615, 0.0004599907260853797, 0.0, 0.0], [0.03717399016022682, 0.005006894003599882, 0.0016112301964312792, 0.0028301470447331667, 0.0005118494154885411, 0.0027071270160377026, 0.0020970311015844345, 0.0012328773736953735, 0.0006310884491540492, 0.001763244392350316, 0.001786878565326333, 0.00025283300783485174, 0.0010720754507929087, 0.0026448338758200407, 0.0015661576762795448, 0.0038084082771092653, 0.005276741925626993, 0.006930254865437746, 0.01331083569675684, 0.0035816438030451536, 0.018304521217942238, 0.026684748008847237, 0.019971538335084915, 0.03215576708316803, 0.07432040572166443, 0.023922255262732506, 0.08761478215456009, 0.1800757199525833, 0.027492573484778404, 0.059722453355789185, 0.19122688472270966, 0.1627122461795807, 0.0], [0.00811693910509348, 0.0008723069913685322, 0.0003707293944898993, 0.00047348643420264125, 0.0001783298939699307, 0.0003994710568804294, 0.00029511720640584826, 0.00018311945314053446, 0.000167145932209678, 0.00023702479666098952, 0.0002533360675442964, 0.00012855060049332678, 0.00020213205425534397, 0.0003994489670731127, 0.0001651181373745203, 0.00046228704741224647, 0.0008780225180089474, 0.0011140474816784263, 0.0020708786323666573, 0.0019636773504316807, 0.0035278573632240295, 0.0044752187095582485, 0.004463486839085817, 0.007312532514333725, 0.014446244575083256, 0.035030465573072433, 0.02001415751874447, 0.04388578608632088, 0.019436758011579514, 0.11325737833976746, 0.07277627289295197, 0.16196827590465546, 0.4804745018482208]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.947374701499939, 0.05262523517012596, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8785883784294128, 0.055394671857357025, 0.06601690500974655, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7270770072937012, 0.1370987594127655, 0.04866093024611473, 0.08716330677270889, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4651746451854706, 0.13429315388202667, 0.1535816341638565, 0.1747254729270935, 0.07222513109445572, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1703597903251648, 0.011841950938105583, 0.0236508846282959, 0.021745815873146057, 0.025856325402855873, 0.7465452551841736, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14214572310447693, 0.04383273795247078, 0.050211820751428604, 0.04516506567597389, 0.058721259236335754, 0.5111533999443054, 0.1487700343132019, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0847334936261177, 0.020369820296764374, 0.03746161609888077, 0.03195912018418312, 0.03419598937034607, 0.43500685691833496, 0.1189405769109726, 0.2373325079679489, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3423079550266266, 0.03593634441494942, 0.0725921243429184, 0.10352761298418045, 0.08196527510881424, 0.07694562524557114, 0.14196324348449707, 0.09896710515022278, 0.045794595032930374, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3711485266685486, 0.06278738379478455, 0.05806950107216835, 0.055662769824266434, 0.13124839961528778, 0.07535629719495773, 0.10726629197597504, 0.07924024015665054, 0.030107343569397926, 0.029113225638866425, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.06430108100175858, 0.0051691061817109585, 0.01101299561560154, 0.009054888971149921, 0.011748063378036022, 0.3188501000404358, 0.03684947267174721, 0.0831955149769783, 0.012068143114447594, 0.014676210470497608, 0.433074414730072, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.23777922987937927, 0.03812997415661812, 0.01974336989223957, 0.07508028298616409, 0.14420795440673828, 0.06244930252432823, 0.07018385827541351, 0.06523600965738297, 0.048832181841135025, 0.03460032865405083, 0.06439767777919769, 0.13935980200767517, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.29014477133750916, 0.04085766524076462, 0.06965630501508713, 0.06467737257480621, 0.11515594273805618, 0.07279439270496368, 0.05406038090586662, 0.040959976613521576, 0.036461226642131805, 0.03709695488214493, 0.08112155646085739, 0.06249447911977768, 0.03451891988515854, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04191606491804123, 0.0033880723640322685, 0.007082367315888405, 0.005899207200855017, 0.007586182560771704, 0.20389790832996368, 0.023296764120459557, 0.05173072963953018, 0.007816805504262447, 0.00973137654364109, 0.2776019871234894, 0.008386771194636822, 0.01617271639406681, 0.3354930877685547, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15129348635673523, 0.04240793362259865, 0.043833814561367035, 0.09706760942935944, 0.14943556487560272, 0.044398821890354156, 0.0617450475692749, 0.04199514910578728, 0.018608780577778816, 0.031412459909915924, 0.05092642828822136, 0.06660290062427521, 0.09416216611862183, 0.057612884789705276, 0.04849693179130554, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.29958024621009827, 0.07432066649198532, 0.03006197139620781, 0.016736475750803947, 0.038564808666706085, 0.05106877535581589, 0.09408668428659439, 0.05606791749596596, 0.054582275450229645, 0.045220401138067245, 0.056380707770586014, 0.015378053300082684, 0.04082852229475975, 0.06084694340825081, 0.030811049044132233, 0.03546451777219772, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.030439963564276695, 0.0024933936074376106, 0.005298204720020294, 0.004223393276333809, 0.005517445504665375, 0.14508457481861115, 0.01603415422141552, 0.03598647937178612, 0.0057580480352044106, 0.007398170419037342, 0.19613467156887054, 0.006090890150517225, 0.011465118266642094, 0.23737992346286774, 0.008031771518290043, 0.005861491896212101, 0.2768023610115051, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015668945387005806, 0.0019133484456688166, 0.006747933570295572, 0.005610871594399214, 0.006710356567054987, 0.10858529061079025, 0.013436974957585335, 0.030674796551465988, 0.00869804061949253, 0.005958769004791975, 0.15696409344673157, 0.00640126271173358, 0.01435719896107912, 0.19220466911792755, 0.009938806295394897, 0.008154557086527348, 0.22861403226852417, 0.1793600618839264, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2751207649707794, 0.046356070786714554, 0.025782747194170952, 0.02037108689546585, 0.04519129917025566, 0.047415051609277725, 0.05800836905837059, 0.054494574666023254, 0.02185583859682083, 0.028674187138676643, 0.05054797977209091, 0.03578946739435196, 0.014343999326229095, 0.05505189672112465, 0.019970891997218132, 0.0250706784427166, 0.05983670800924301, 0.10591268539428711, 0.01020571868866682, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12210460752248764, 0.029006367549300194, 0.02792610600590706, 0.030763691291213036, 0.06431955099105835, 0.05075869709253311, 0.05746712535619736, 0.03854454681277275, 0.02881445176899433, 0.03641033545136452, 0.05530688911676407, 0.02755570411682129, 0.05062380060553551, 0.06039686128497124, 0.03790910169482231, 0.034162603318691254, 0.06550325453281403, 0.08224605768918991, 0.08903136104345322, 0.011148839257657528, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01992272026836872, 0.0015991260297596455, 0.003536272095516324, 0.0027606096118688583, 0.0036747483536601067, 0.09198574721813202, 0.009868193417787552, 0.021762730553746223, 0.0036746191326528788, 0.004800298251211643, 0.12327182292938232, 0.003948224242776632, 0.007727704476565123, 0.14901942014694214, 0.005094495136290789, 0.0038495466578751802, 0.17433811724185944, 0.1450374871492386, 0.003139176871627569, 0.0039015670772641897, 0.21708732843399048, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.025169141590595245, 0.0025891433469951153, 0.007904480211436749, 0.002818175358697772, 0.008317488245666027, 0.06284766644239426, 0.016097376123070717, 0.024833567440509796, 0.007658041547983885, 0.011317074298858643, 0.08451006561517715, 0.012135406024754047, 0.022048376500606537, 0.1001725122332573, 0.015103276818990707, 0.020152254030108452, 0.11906814575195312, 0.141489639878273, 0.006576653569936752, 0.007074056193232536, 0.1485065221786499, 0.1536109447479248, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08679644763469696, 0.01581868715584278, 0.009913153015077114, 0.005941125564277172, 0.01174909994006157, 0.04402247071266174, 0.0494115874171257, 0.047211576253175735, 0.022448517382144928, 0.02814415469765663, 0.05249372497200966, 0.013664884492754936, 0.0380445197224617, 0.05856490507721901, 0.01600872538983822, 0.028531765565276146, 0.06752399355173111, 0.1000405102968216, 0.026074493303894997, 0.01816527359187603, 0.08061447739601135, 0.14588385820388794, 0.03293200954794884, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12203323096036911, 0.02679692953824997, 0.03471272066235542, 0.016600849106907845, 0.05098596587777138, 0.033623479306697845, 0.03376290947198868, 0.03289441019296646, 0.022763589397072792, 0.03469543159008026, 0.03710823133587837, 0.02750304900109768, 0.030508041381835938, 0.04070492461323738, 0.019409826025366783, 0.033388134092092514, 0.046037882566452026, 0.08141940832138062, 0.025792937725782394, 0.017713353037834167, 0.05326378345489502, 0.06955216079950333, 0.07340100407600403, 0.03532780334353447, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.012632706202566624, 0.003196730511263013, 0.00899962242692709, 0.003787106368690729, 0.0055902148596942425, 0.060187432914972305, 0.010173025541007519, 0.026063401252031326, 0.008280332200229168, 0.005998437758535147, 0.08480245620012283, 0.007755732629448175, 0.014807655476033688, 0.10079342871904373, 0.011636020615696907, 0.007885102182626724, 0.120613694190979, 0.12819863855838776, 0.006487111561000347, 0.00323057291097939, 0.15047506988048553, 0.10557688027620316, 0.015480604022741318, 0.01902633160352707, 0.0783216655254364, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11176053434610367, 0.018635563552379608, 0.05622374638915062, 0.046772800385951996, 0.03736491501331329, 0.02299342304468155, 0.031365279108285904, 0.026408886536955833, 0.043359220027923584, 0.056635573506355286, 0.021786397323012352, 0.023822149261832237, 0.030641673132777214, 0.023289257660508156, 0.05430489778518677, 0.06881187111139297, 0.024489382281899452, 0.04777085781097412, 0.036271754652261734, 0.022222526371479034, 0.026998935267329216, 0.02176610752940178, 0.03442488983273506, 0.028240257874131203, 0.05891920626163483, 0.024719828739762306, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10639805346727371, 0.0371079258620739, 0.08273956179618835, 0.01785685308277607, 0.020949997007846832, 0.018305253237485886, 0.02262011356651783, 0.018192270770668983, 0.021851180121302605, 0.01251852884888649, 0.019214097410440445, 0.03034009225666523, 0.0718761533498764, 0.020787999033927917, 0.03963525965809822, 0.06151590868830681, 0.02317928522825241, 0.0439804382622242, 0.04619227349758148, 0.02066798508167267, 0.027080876752734184, 0.04386550188064575, 0.0327674001455307, 0.04369068145751953, 0.05080847069621086, 0.04029490798711777, 0.025563018396496773, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015084554441273212, 0.003822384402155876, 0.005877268500626087, 0.0049718520604074, 0.005370147991925478, 0.05223645642399788, 0.016127675771713257, 0.03182988613843918, 0.00821112934499979, 0.011473809368908405, 0.06704685091972351, 0.010528381913900375, 0.010598219931125641, 0.07938898354768753, 0.007211714517325163, 0.0062714628875255585, 0.09142373502254486, 0.10866505652666092, 0.003973816987127066, 0.005621673539280891, 0.1098332554101944, 0.09237755835056305, 0.02296048402786255, 0.019314926117658615, 0.08864143490791321, 0.018464695662260056, 0.011022249236702919, 0.0916503295302391, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09854131937026978, 0.03967057913541794, 0.05052143707871437, 0.04794390872120857, 0.0195318553596735, 0.015683406963944435, 0.02507476694881916, 0.014997915364801884, 0.02772345580160618, 0.016418185085058212, 0.016389017924666405, 0.026316562667489052, 0.03435927629470825, 0.017757989466190338, 0.0233626626431942, 0.03763514757156372, 0.019405605271458626, 0.034421682357788086, 0.04902297630906105, 0.030190544202923775, 0.022159066051244736, 0.037540629506111145, 0.038793135434389114, 0.04032415151596069, 0.0423126220703125, 0.06829195469617844, 0.025314129889011383, 0.04330073297023773, 0.03699527308344841, 0.0, 0.0, 0.0, 0.0], [0.10688814520835876, 0.03659575805068016, 0.005083094350993633, 0.01016042660921812, 0.013297352008521557, 0.04175207018852234, 0.0262468159198761, 0.03012855537235737, 0.013482386246323586, 0.018223188817501068, 0.0458262674510479, 0.02555879019200802, 0.015618832781910896, 0.05039895698428154, 0.01165701076388359, 0.011659128591418266, 0.0551551915705204, 0.0509166419506073, 0.019206784665584564, 0.013998407870531082, 0.06383473426103592, 0.02908892184495926, 0.016710441559553146, 0.02609526738524437, 0.05484986677765846, 0.03806670755147934, 0.03896071016788483, 0.05375232174992561, 0.023679165169596672, 0.0531080923974514, 0.0, 0.0, 0.0], [0.10871604084968567, 0.06286415457725525, 0.02304510399699211, 0.012493208050727844, 0.009688262827694416, 0.02372078038752079, 0.025617338716983795, 0.028282862156629562, 0.019523099064826965, 0.015187123790383339, 0.024615760892629623, 0.018855812028050423, 0.013071928173303604, 0.02569747343659401, 0.010639789514243603, 0.007336440030485392, 0.028488244861364365, 0.040875114500522614, 0.023693937808275223, 0.02846083790063858, 0.031781155616045, 0.03399210050702095, 0.01690451242029667, 0.04699751362204552, 0.054809220135211945, 0.0604817196726799, 0.021137842908501625, 0.05147771164774895, 0.01530434750020504, 0.07868867367506027, 0.03755194693803787, 0.0, 0.0], [0.16510623693466187, 0.013307320885360241, 0.022820336744189262, 0.032224975526332855, 0.028327813372015953, 0.011912115849554539, 0.021759122610092163, 0.012591322883963585, 0.025371331721544266, 0.01592349074780941, 0.011745063588023186, 0.021340155974030495, 0.022658517584204674, 0.012096998281776905, 0.05877390503883362, 0.041142962872982025, 0.012866133823990822, 0.02189669944345951, 0.02738575078547001, 0.015412718988955021, 0.014822259545326233, 0.013198777101933956, 0.029060106724500656, 0.04031305015087128, 0.023036926984786987, 0.05035790055990219, 0.046500690281391144, 0.03221071884036064, 0.04574136063456535, 0.011448168195784092, 0.05148483067750931, 0.04716223478317261, 0.0], [0.019862741231918335, 0.0006941381143406034, 0.0017670642118901014, 0.0019038202008232474, 0.0023115782532840967, 0.05924388766288757, 0.005164953414350748, 0.008714546449482441, 0.0024906685575842857, 0.0030579573940485716, 0.07585158944129944, 0.002390549285337329, 0.0031773503869771957, 0.08945121616125107, 0.0030652948189526796, 0.0016209169989451766, 0.10312031954526901, 0.0625915676355362, 0.002058295765891671, 0.002232219558209181, 0.12742049992084503, 0.024703850969672203, 0.0035101925022900105, 0.00499417819082737, 0.028317121788859367, 0.005874201189726591, 0.006215325556695461, 0.03686564415693283, 0.005438121035695076, 0.004061868414282799, 0.004735523369163275, 0.004534382838755846, 0.2925584614276886]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9726406335830688, 0.027359364554286003, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8850566744804382, 0.08160645514726639, 0.03333686292171478, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7844979166984558, 0.09045999497175217, 0.10720972716808319, 0.01783234067261219, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6679065227508545, 0.10135099291801453, 0.10997921228408813, 0.0699087604880333, 0.050854478031396866, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.586279034614563, 0.06960930675268173, 0.06798022985458374, 0.06058962643146515, 0.08339767158031464, 0.13214409351348877, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5015710592269897, 0.06227877363562584, 0.07209988683462143, 0.05590919405221939, 0.08146880567073822, 0.12033165246248245, 0.10634062439203262, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.40016087889671326, 0.05525398626923561, 0.06347683072090149, 0.05413786694407463, 0.07119617611169815, 0.11230527609586716, 0.12489721924066544, 0.11857166141271591, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4018532931804657, 0.08579416573047638, 0.056448839604854584, 0.07181122153997421, 0.08318912237882614, 0.07845646142959595, 0.09191881120204926, 0.07768876850605011, 0.05283927917480469, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3827159106731415, 0.07976946234703064, 0.062077559530735016, 0.06360235810279846, 0.08211023360490799, 0.07804954797029495, 0.08597437292337418, 0.07506486773490906, 0.0632784441113472, 0.02735721692442894, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3404167592525482, 0.04535197094082832, 0.04658910632133484, 0.0403917022049427, 0.05489042401313782, 0.08179006725549698, 0.08906516432762146, 0.09645897150039673, 0.05564969405531883, 0.06398677080869675, 0.08540938794612885, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.31186187267303467, 0.09539996832609177, 0.07292013615369797, 0.0697687417268753, 0.07777827978134155, 0.06735038757324219, 0.06161406263709068, 0.054810743778944016, 0.052520137280225754, 0.05022520199418068, 0.07299348711967468, 0.012756960466504097, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.36652863025665283, 0.06308672577142715, 0.03947845473885536, 0.04609525203704834, 0.05740750953555107, 0.0634833425283432, 0.05238465964794159, 0.05501618608832359, 0.04584015905857086, 0.06228693574666977, 0.0658990740776062, 0.058063607662916183, 0.024429509416222572, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.28564438223838806, 0.03804025053977966, 0.03942390903830528, 0.034238625317811966, 0.0463939793407917, 0.06771662831306458, 0.07328074425458908, 0.07855433225631714, 0.047896627336740494, 0.055936526507139206, 0.07124947756528854, 0.035689257085323334, 0.04956042021512985, 0.07637479156255722, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.297754168510437, 0.05259547755122185, 0.04347941279411316, 0.05845862627029419, 0.04568730294704437, 0.05259405076503754, 0.04456199333071709, 0.0459747239947319, 0.06316147744655609, 0.0776875764131546, 0.05499143525958061, 0.03466471657156944, 0.05020103231072426, 0.05886140838265419, 0.01932653971016407, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.28853145241737366, 0.04616911709308624, 0.028233066201210022, 0.038242388516664505, 0.04330621287226677, 0.05174916982650757, 0.04318275675177574, 0.043931201100349426, 0.05616341903805733, 0.06773821264505386, 0.05490177869796753, 0.08139309287071228, 0.04325690492987633, 0.0590481236577034, 0.03989904746413231, 0.014254068955779076, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.23721373081207275, 0.03161581605672836, 0.03324565663933754, 0.029231945052742958, 0.038618117570877075, 0.05549168214201927, 0.05966496840119362, 0.06321781128644943, 0.040736980736255646, 0.04752674326300621, 0.058378204703330994, 0.03039967454969883, 0.04287158325314522, 0.06284616142511368, 0.053212910890579224, 0.04736898094415665, 0.06835903972387314, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2098696380853653, 0.02729126438498497, 0.031156614422798157, 0.026615004986524582, 0.03165113925933838, 0.05273226648569107, 0.052745021879673004, 0.05635902285575867, 0.04076546058058739, 0.04278237372636795, 0.056191232055425644, 0.030678272247314453, 0.04398961737751961, 0.06091473996639252, 0.0501369833946228, 0.0415731742978096, 0.06652864068746567, 0.07801953703165054, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2385382503271103, 0.04414580762386322, 0.046435050666332245, 0.0345127172768116, 0.0357217937707901, 0.04369329661130905, 0.04304974898695946, 0.034988246858119965, 0.043718066066503525, 0.0395219661295414, 0.044578537344932556, 0.024599451571702957, 0.0625118762254715, 0.04753483459353447, 0.043756093829870224, 0.04995930567383766, 0.050908081233501434, 0.0526624359190464, 0.019164469093084335, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.17729151248931885, 0.042214617133140564, 0.030818725004792213, 0.030886096879839897, 0.033868495374917984, 0.045211877673864365, 0.042901039123535156, 0.032215580344200134, 0.040188658982515335, 0.049268968403339386, 0.04801624268293381, 0.03814242035150528, 0.051265135407447815, 0.05206841975450516, 0.047441162168979645, 0.03924533724784851, 0.05630478635430336, 0.05462942644953728, 0.0599186010658741, 0.02810293436050415, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19048276543617249, 0.025203485041856766, 0.026787471026182175, 0.02367268316447735, 0.03110402822494507, 0.04345344379544258, 0.04649331048130989, 0.048206478357315063, 0.03323280066251755, 0.03892753645777702, 0.04549149051308632, 0.02444273605942726, 0.035701580345630646, 0.04907767102122307, 0.04303564876317978, 0.039055656641721725, 0.0537174753844738, 0.06658957898616791, 0.04320826008915901, 0.03176470100879669, 0.06035127118229866, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14975358545780182, 0.023474570363759995, 0.025508398190140724, 0.028942441567778587, 0.03517158329486847, 0.038801901042461395, 0.04594280198216438, 0.03962729126214981, 0.037087779492139816, 0.038723014295101166, 0.04190719500184059, 0.021688053384423256, 0.04328255355358124, 0.04592541232705116, 0.03893951326608658, 0.0400761142373085, 0.05060023441910744, 0.060830533504486084, 0.043721601366996765, 0.03023521602153778, 0.05728611722588539, 0.062474027276039124, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.17513407766819, 0.027519281953573227, 0.029514014720916748, 0.02071532793343067, 0.026873527094721794, 0.0345461331307888, 0.038530781865119934, 0.033037811517715454, 0.04181556776165962, 0.04010028764605522, 0.03696895390748978, 0.030468937009572983, 0.039953943341970444, 0.04023702070116997, 0.0448262095451355, 0.03708262741565704, 0.04420144855976105, 0.04931037127971649, 0.04125744849443436, 0.028610678389668465, 0.04997344687581062, 0.05269285663962364, 0.036629173904657364, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13285990059375763, 0.040708061307668686, 0.04111678898334503, 0.034844279289245605, 0.04087604954838753, 0.027567215263843536, 0.028293700888752937, 0.022520417347550392, 0.0387541837990284, 0.040378037840127945, 0.029089409857988358, 0.029609227553009987, 0.0567011833190918, 0.031181858852505684, 0.037661172449588776, 0.056886378675699234, 0.03396708890795708, 0.037813056260347366, 0.04951409250497818, 0.031945571303367615, 0.037599027156829834, 0.04326637089252472, 0.043056681752204895, 0.033790234476327896, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14709368348121643, 0.018218709155917168, 0.022143620997667313, 0.02162139303982258, 0.02707289345562458, 0.03385418280959129, 0.03787611797451973, 0.03801965340971947, 0.027932504191994667, 0.03090183064341545, 0.03606922924518585, 0.01902313157916069, 0.03465801104903221, 0.03916986659169197, 0.033755578100681305, 0.02613663300871849, 0.042978327721357346, 0.05313742905855179, 0.03463505581021309, 0.02014029398560524, 0.04878358915448189, 0.04990803450345993, 0.04624392092227936, 0.04715671017765999, 0.06346960365772247, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1760278195142746, 0.01488215196877718, 0.026751182973384857, 0.0240880586206913, 0.03566748648881912, 0.027094293385744095, 0.02498425543308258, 0.021471885964274406, 0.026359373703598976, 0.023845119401812553, 0.027566982433199883, 0.01686953380703926, 0.0777742937207222, 0.029109211638569832, 0.04799698293209076, 0.02900250256061554, 0.03132336959242821, 0.03615599498152733, 0.03959561511874199, 0.017008014023303986, 0.034839075058698654, 0.03443719446659088, 0.04149218648672104, 0.09124380350112915, 0.040522992610931396, 0.0038906026165932417, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1850174218416214, 0.027787016704678535, 0.025097640231251717, 0.027018999680876732, 0.029199207201600075, 0.030337024480104446, 0.032221775501966476, 0.02301507070660591, 0.0267928596585989, 0.022793851792812347, 0.030674852430820465, 0.021220838651061058, 0.053888075053691864, 0.03259299322962761, 0.030737651512026787, 0.025725245475769043, 0.03486606478691101, 0.037935830652713776, 0.028672009706497192, 0.026148982346057892, 0.03875335678458214, 0.04179506003856659, 0.03863157331943512, 0.05337152257561684, 0.03982071578502655, 0.02571161463856697, 0.010172729380428791, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12446325272321701, 0.01676889881491661, 0.02010510489344597, 0.019267024472355843, 0.024457814171910286, 0.03020336478948593, 0.03127393499016762, 0.03254607319831848, 0.023186003789305687, 0.02350839227437973, 0.0317331999540329, 0.017080510035157204, 0.029082054272294044, 0.03428216651082039, 0.03579127788543701, 0.025820568203926086, 0.037531185895204544, 0.04618249833583832, 0.029500003904104233, 0.017913807183504105, 0.04274731129407883, 0.042434703558683395, 0.039002396166324615, 0.04141838476061821, 0.05949503183364868, 0.03595735505223274, 0.030984872952103615, 0.057262834161520004, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14491021633148193, 0.025843990966677666, 0.031176971271634102, 0.01946275867521763, 0.012575727887451649, 0.025274591520428658, 0.022020690143108368, 0.021267279982566833, 0.03165539726614952, 0.02766416408121586, 0.026251589879393578, 0.022091038525104523, 0.03851794824004173, 0.028266867622733116, 0.0288776233792305, 0.036865364760160446, 0.03069997765123844, 0.03204502537846565, 0.03411627188324928, 0.02035541459918022, 0.03446315601468086, 0.03542865440249443, 0.030728410929441452, 0.04441893845796585, 0.03793596476316452, 0.05734865367412567, 0.03889700025320053, 0.04111205041408539, 0.019728200510144234, 0.0, 0.0, 0.0, 0.0], [0.13168425858020782, 0.025460105389356613, 0.02796240895986557, 0.024109845981001854, 0.01686880923807621, 0.031406790018081665, 0.023247336968779564, 0.022071687504649162, 0.027362508699297905, 0.02637045457959175, 0.032045334577560425, 0.02877580001950264, 0.04614202678203583, 0.03455977141857147, 0.03132109344005585, 0.02943773753941059, 0.03749070689082146, 0.03597421571612358, 0.0306710172444582, 0.026778314262628555, 0.041662365198135376, 0.03274398297071457, 0.030348768457770348, 0.032651983201503754, 0.038056500256061554, 0.034575607627630234, 0.022936122491955757, 0.03690328821539879, 0.02596619911491871, 0.01441489439457655, 0.0, 0.0, 0.0], [0.1656370609998703, 0.02280924655497074, 0.029107755050063133, 0.02494465932250023, 0.02435287833213806, 0.02503182739019394, 0.018486661836504936, 0.01857367716729641, 0.028443630784749985, 0.029199941083788872, 0.0253182053565979, 0.03220498561859131, 0.025753282010555267, 0.026890520006418228, 0.031273938715457916, 0.03305148705840111, 0.028835458680987358, 0.029910407960414886, 0.0275588259100914, 0.024624649435281754, 0.0320536270737648, 0.02649376355111599, 0.02170960046350956, 0.031640153378248215, 0.03707476332783699, 0.0479947030544281, 0.02793215773999691, 0.034756116569042206, 0.03804944083094597, 0.02153567597270012, 0.008750851266086102, 0.0, 0.0], [0.16794215142726898, 0.025983478873968124, 0.03222338482737541, 0.019469385966658592, 0.023735277354717255, 0.023115742951631546, 0.0199736300855875, 0.016550231724977493, 0.029796775430440903, 0.028494667261838913, 0.023108843713998795, 0.02208707109093666, 0.042165566235780716, 0.02439018338918686, 0.02074914425611496, 0.038735877722501755, 0.026269812136888504, 0.02728910744190216, 0.019617389887571335, 0.020242862403392792, 0.02920321375131607, 0.030413366854190826, 0.02576262876391411, 0.040849827229976654, 0.033326659351587296, 0.021795300766825676, 0.026310516521334648, 0.03101349249482155, 0.03689264506101608, 0.01937715709209442, 0.04355606809258461, 0.009558655321598053, 0.0], [0.11396828293800354, 0.01775287464261055, 0.01606196165084839, 0.015585055574774742, 0.01839612051844597, 0.024367082864046097, 0.026053881272673607, 0.023999109864234924, 0.021919487044215202, 0.026251113042235374, 0.024730533361434937, 0.015323017723858356, 0.025614570826292038, 0.0263378843665123, 0.024982867762446404, 0.02464262954890728, 0.028590328991413116, 0.03295150771737099, 0.027568619698286057, 0.01848754845559597, 0.03220851346850395, 0.031914692372083664, 0.036307260394096375, 0.0341540202498436, 0.036785583943128586, 0.03297772631049156, 0.026278911158442497, 0.03809891268610954, 0.029801947996020317, 0.018183261156082153, 0.03632068261504173, 0.035680945962667465, 0.05770310387015343]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8058414459228516, 0.19415856897830963, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6373865008354187, 0.09672051668167114, 0.26589304208755493, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5870265364646912, 0.09922721236944199, 0.07913831621408463, 0.23460792005062103, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4194737374782562, 0.06312593817710876, 0.08123067021369934, 0.08381500095129013, 0.35235461592674255, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5308531522750854, 0.0959072858095169, 0.07779339700937271, 0.06661548465490341, 0.07293295115232468, 0.15589767694473267, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.38503071665763855, 0.07478094846010208, 0.06978150457143784, 0.047606538981199265, 0.051449161022901535, 0.08776707202196121, 0.2835841178894043, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2962956726551056, 0.05328270420432091, 0.06944749504327774, 0.04097547382116318, 0.06660831719636917, 0.1021856963634491, 0.10869503766298294, 0.26250961422920227, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.30474990606307983, 0.06337985396385193, 0.05753423646092415, 0.04057719185948372, 0.0730014219880104, 0.07375827431678772, 0.07239888608455658, 0.05693604797124863, 0.25766414403915405, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.29880762100219727, 0.05552824214100838, 0.05443597584962845, 0.05285302922129631, 0.06568750739097595, 0.07554222643375397, 0.06968103349208832, 0.054113879799842834, 0.09120790660381317, 0.18214264512062073, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.26933014392852783, 0.06352431327104568, 0.059521228075027466, 0.052424702793359756, 0.06052157282829285, 0.12185832113027573, 0.07608654350042343, 0.09753072261810303, 0.04232607036828995, 0.05058327317237854, 0.10629311949014664, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.22074301540851593, 0.06960077583789825, 0.048723481595516205, 0.06675717234611511, 0.06954211741685867, 0.06483786553144455, 0.046523742377758026, 0.039300836622714996, 0.018016092479228973, 0.0358063168823719, 0.053022876381874084, 0.26712578535079956, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15080073475837708, 0.04732475057244301, 0.12551115453243256, 0.06240082159638405, 0.05354631692171097, 0.05578393489122391, 0.04542611539363861, 0.03888600319623947, 0.0301998108625412, 0.033370546996593475, 0.04645953327417374, 0.023437919095158577, 0.2868523597717285, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2044883519411087, 0.04972049966454506, 0.04838546738028526, 0.043853580951690674, 0.05115717276930809, 0.1007489338517189, 0.06420683115720749, 0.08302561193704605, 0.03880453109741211, 0.047957729548215866, 0.09582382440567017, 0.03239700198173523, 0.04441104456782341, 0.0950193703174591, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1623673439025879, 0.028519174084067345, 0.047843094915151596, 0.041354816406965256, 0.040284883230924606, 0.0520053468644619, 0.04226505383849144, 0.03821168839931488, 0.05986882373690605, 0.06849399209022522, 0.04734278842806816, 0.019715160131454468, 0.06202228367328644, 0.04572192579507828, 0.24398362636566162, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13128933310508728, 0.039628103375434875, 0.059519413858652115, 0.03966323286294937, 0.04406355693936348, 0.04348496347665787, 0.03726547583937645, 0.03240034729242325, 0.03720574453473091, 0.06481601297855377, 0.040474534034729004, 0.025914235040545464, 0.06507598608732224, 0.03922511264681816, 0.07744196802377701, 0.2225320190191269, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15813341736793518, 0.03911251947283745, 0.03856496512889862, 0.03587968647480011, 0.04187794774770737, 0.08161000907421112, 0.05245806276798248, 0.0681268721818924, 0.03444807231426239, 0.04342752322554588, 0.08275996893644333, 0.02933340333402157, 0.041573796421289444, 0.08478686958551407, 0.04881599545478821, 0.032352838665246964, 0.08673813939094543, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13890595734119415, 0.03169632703065872, 0.03251556307077408, 0.03538898378610611, 0.038000334054231644, 0.06974425166845322, 0.059424545615911484, 0.06768239289522171, 0.027843810617923737, 0.039054665714502335, 0.07273104041814804, 0.030109861865639687, 0.04163092002272606, 0.07565011829137802, 0.040188319981098175, 0.027483616024255753, 0.07795906811952591, 0.09399020671844482, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09691290557384491, 0.030687319114804268, 0.030817966908216476, 0.04473433643579483, 0.04171944037079811, 0.036036767065525055, 0.04101703688502312, 0.032104767858982086, 0.04344063252210617, 0.048855531960725784, 0.03457901254296303, 0.016025088727474213, 0.049941662698984146, 0.0346352756023407, 0.060656867921352386, 0.034977883100509644, 0.03454342484474182, 0.037827931344509125, 0.25048619508743286, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12059216946363449, 0.04121065139770508, 0.0509851798415184, 0.052891526371240616, 0.02886129543185234, 0.04706199839711189, 0.0292128287255764, 0.038170017302036285, 0.021046487614512444, 0.029787806794047356, 0.04406721517443657, 0.02650333382189274, 0.04127003997564316, 0.04430737346410751, 0.03080155700445175, 0.04064903408288956, 0.044745106250047684, 0.042122457176446915, 0.02844228595495224, 0.1972716897726059, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11388304829597473, 0.028255397453904152, 0.028351297602057457, 0.026958664879202843, 0.031310852617025375, 0.05977548286318779, 0.0391821525990963, 0.05072040110826492, 0.027684014290571213, 0.035564228892326355, 0.06452134251594543, 0.02426709607243538, 0.03616495057940483, 0.06838168203830719, 0.04139300435781479, 0.029094640165567398, 0.0723004937171936, 0.07349380850791931, 0.037000205367803574, 0.0352863185107708, 0.07641094923019409, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08453849703073502, 0.017416605725884438, 0.01894914172589779, 0.0165606327354908, 0.02229611948132515, 0.04174060747027397, 0.05894864723086357, 0.03893362358212471, 0.030214650556445122, 0.034642331302165985, 0.0483190156519413, 0.016358422115445137, 0.02567455545067787, 0.05309673771262169, 0.02054651454091072, 0.02046983130276203, 0.057046402245759964, 0.06604909896850586, 0.03774644806981087, 0.021413879469037056, 0.06129930540919304, 0.2077389657497406, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1124843955039978, 0.024394847452640533, 0.024523155763745308, 0.027489561587572098, 0.026104671880602837, 0.03731505200266838, 0.06748531758785248, 0.03890375792980194, 0.01798143796622753, 0.028454387560486794, 0.03756532073020935, 0.01832606829702854, 0.020088663324713707, 0.03865170478820801, 0.0232856422662735, 0.024226617068052292, 0.03960612043738365, 0.04833188280463219, 0.04500093311071396, 0.017362147569656372, 0.04009140655398369, 0.05208287015557289, 0.19024400413036346, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08183357119560242, 0.019828449934720993, 0.02687339298427105, 0.029252557083964348, 0.02518443763256073, 0.03192773088812828, 0.029049793258309364, 0.024972520768642426, 0.03350626677274704, 0.04007429629564285, 0.033517319709062576, 0.016035988926887512, 0.03204454854130745, 0.03448064625263214, 0.028517093509435654, 0.03532731533050537, 0.03583228215575218, 0.04058396443724632, 0.029941106215119362, 0.02109791524708271, 0.03671329841017723, 0.039212584495544434, 0.03543241322040558, 0.23876062035560608, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10145692527294159, 0.014237047173082829, 0.016984183341264725, 0.018636571243405342, 0.02853623777627945, 0.0365380235016346, 0.02957051247358322, 0.041983190923929214, 0.014166045933961868, 0.030081408098340034, 0.04016641527414322, 0.02007502317428589, 0.030370114371180534, 0.043534401804208755, 0.030393019318580627, 0.020717334002256393, 0.04677111282944679, 0.0549510233104229, 0.028809182345867157, 0.020043687894940376, 0.05092543363571167, 0.04084467142820358, 0.0418580137193203, 0.03848976269364357, 0.15986064076423645, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07937924563884735, 0.020674310624599457, 0.020799757912755013, 0.027988232672214508, 0.04565475508570671, 0.02656259387731552, 0.02539670839905739, 0.021243257448077202, 0.01776379905641079, 0.02520064264535904, 0.027576463297009468, 0.030607089400291443, 0.034380000084638596, 0.028568314388394356, 0.022726934403181076, 0.01605195552110672, 0.029712749645113945, 0.03276180848479271, 0.025702400133013725, 0.012878884561359882, 0.030626127496361732, 0.02036965638399124, 0.03741392120718956, 0.027850378304719925, 0.035718515515327454, 0.27639153599739075, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09605434536933899, 0.02309289574623108, 0.024199659004807472, 0.024974144995212555, 0.036829639226198196, 0.03001439943909645, 0.021069226786494255, 0.018117526546120644, 0.022569267079234123, 0.02834322676062584, 0.030505143105983734, 0.021707680076360703, 0.028211787343025208, 0.031639426946640015, 0.03916612267494202, 0.03627365827560425, 0.03311571851372719, 0.03514396771788597, 0.02574523724615574, 0.015889015048742294, 0.034675464034080505, 0.02677008882164955, 0.027143454179167747, 0.0404508002102375, 0.037603847682476044, 0.032323356717824936, 0.17837099730968475, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07476937770843506, 0.011021512560546398, 0.010706555098295212, 0.015325457789003849, 0.01990356110036373, 0.03285866603255272, 0.021936072036623955, 0.02945413999259472, 0.01642150804400444, 0.022476578131318092, 0.03722343221306801, 0.01758023351430893, 0.021954145282506943, 0.040462132543325424, 0.031415410339832306, 0.010417776182293892, 0.044297393411397934, 0.05118127167224884, 0.029565559700131416, 0.015394334681332111, 0.0490642674267292, 0.03202195093035698, 0.030905120074748993, 0.04132053256034851, 0.06578273326158524, 0.03978404775261879, 0.02049970254302025, 0.16625656187534332, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05124205723404884, 0.011390028521418571, 0.02026580646634102, 0.024529647082090378, 0.12732437252998352, 0.020083913579583168, 0.017692744731903076, 0.020381992682814598, 0.018548619002103806, 0.023206066340208054, 0.021026061847805977, 0.020521236583590508, 0.025210734456777573, 0.021994242444634438, 0.017190473154187202, 0.02080591395497322, 0.023035308346152306, 0.026458054780960083, 0.031086130067706108, 0.011454472318291664, 0.024357995018363, 0.02252495475113392, 0.024600783362984657, 0.024588001891970634, 0.02838081493973732, 0.04660612717270851, 0.043627094477415085, 0.02030046284198761, 0.21156589686870575, 0.0, 0.0, 0.0, 0.0], [0.07929293811321259, 0.023537395521998405, 0.022847289219498634, 0.01993604749441147, 0.015462876297533512, 0.025110295042395592, 0.021429050713777542, 0.015073496848344803, 0.00992409698665142, 0.015724744647741318, 0.027188707143068314, 0.020747287198901176, 0.03497635945677757, 0.029102325439453125, 0.021219033747911453, 0.01814761571586132, 0.03105819784104824, 0.03380950167775154, 0.020137565210461617, 0.012323119677603245, 0.0336926244199276, 0.02433612570166588, 0.031084517017006874, 0.03377701714634895, 0.04355444014072418, 0.032963525503873825, 0.03314930200576782, 0.02536490187048912, 0.022163692861795425, 0.2228659689426422, 0.0, 0.0, 0.0], [0.10059655457735062, 0.03096376359462738, 0.025578510016202927, 0.0202240701764822, 0.027132991701364517, 0.0294907595962286, 0.0407448373734951, 0.021403610706329346, 0.016392609104514122, 0.02252069301903248, 0.027555176988244057, 0.013599206693470478, 0.027306707575917244, 0.027806581929326057, 0.011214177124202251, 0.023586539551615715, 0.02839917503297329, 0.03066891059279442, 0.021664204075932503, 0.017032310366630554, 0.02905159257352352, 0.026549918577075005, 0.03297552466392517, 0.026790324598550797, 0.032060641795396805, 0.02729833871126175, 0.017307184636592865, 0.018768811598420143, 0.028226524591445923, 0.017085110768675804, 0.18000467121601105, 0.0, 0.0], [0.08361606299877167, 0.016581889241933823, 0.015823302790522575, 0.022005166858434677, 0.02406255342066288, 0.021028300747275352, 0.022811293601989746, 0.014412387274205685, 0.01557208877056837, 0.015104830265045166, 0.020917734131217003, 0.021798150613904, 0.02598598226904869, 0.02201320417225361, 0.022025059908628464, 0.018091218546032906, 0.02311510592699051, 0.025451799854636192, 0.02594352327287197, 0.011241885833442211, 0.024466173723340034, 0.0228698942810297, 0.031613439321517944, 0.029542673379182816, 0.02381790056824684, 0.029349016025662422, 0.025922643020749092, 0.017268173396587372, 0.0318446047604084, 0.012037734501063824, 0.027524925768375397, 0.2561413049697876, 0.0], [0.05258440971374512, 0.012508383020758629, 0.012737675569951534, 0.012144193053245544, 0.0144678158685565, 0.024812448769807816, 0.01710914634168148, 0.02516256831586361, 0.016298966482281685, 0.015160080045461655, 0.029820043593645096, 0.014290276914834976, 0.024094797670841217, 0.033742766827344894, 0.021632254123687744, 0.018924105912446976, 0.03819216787815094, 0.04115181416273117, 0.024165764451026917, 0.020635638386011124, 0.044185325503349304, 0.03793709725141525, 0.029636353254318237, 0.03152839466929436, 0.04542306438088417, 0.035963598638772964, 0.030482539907097816, 0.04609914869070053, 0.03206766024231911, 0.01766294240951538, 0.03169601783156395, 0.04462910816073418, 0.10305341333150864]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7707713842391968, 0.22922858595848083, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7813623547554016, 0.109785296022892, 0.10885230451822281, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.535038948059082, 0.11194943636655807, 0.15013694763183594, 0.20287470519542694, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.41803574562072754, 0.1603638231754303, 0.1074029952287674, 0.10625781118869781, 0.20793959498405457, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4349866509437561, 0.09579858183860779, 0.08251801878213882, 0.10028504580259323, 0.15332908928394318, 0.1330825835466385, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.44621357321739197, 0.05624214559793472, 0.05040299892425537, 0.07084295898675919, 0.09360906481742859, 0.11472620815038681, 0.16796305775642395, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3609195351600647, 0.07294464111328125, 0.05191005766391754, 0.07159046083688736, 0.09321668744087219, 0.09543665498495102, 0.14539918303489685, 0.10858277231454849, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3171406686306, 0.07683130353689194, 0.05552283674478531, 0.06445366144180298, 0.08586590737104416, 0.10643255710601807, 0.1452113687992096, 0.09439337253570557, 0.05414833128452301, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19288603961467743, 0.1134386956691742, 0.04530505836009979, 0.08361203223466873, 0.05763273686170578, 0.11914196610450745, 0.11441227793693542, 0.12364513427019119, 0.07640958577394485, 0.07351639866828918, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.25174087285995483, 0.060194116085767746, 0.04930208995938301, 0.059223245829343796, 0.08453336358070374, 0.07307681441307068, 0.12308554351329803, 0.08293969184160233, 0.06380272656679153, 0.07185835391283035, 0.08024313300848007, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2479359656572342, 0.048557743430137634, 0.061716966331005096, 0.06616783887147903, 0.0878860130906105, 0.05920383334159851, 0.09128588438034058, 0.06866813451051712, 0.060628652572631836, 0.08399352431297302, 0.0656941756606102, 0.05826116353273392, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15039603412151337, 0.0707230269908905, 0.036188021302223206, 0.03812884911894798, 0.05496574938297272, 0.09552797675132751, 0.1278020143508911, 0.09101750701665878, 0.04133947193622589, 0.06619426608085632, 0.11493087559938431, 0.03526817262172699, 0.0775180533528328, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.21730750799179077, 0.052248772233724594, 0.04279686510562897, 0.05130988731980324, 0.07021071761846542, 0.06084864214062691, 0.10094168037176132, 0.06729137152433395, 0.05347270891070366, 0.05803239718079567, 0.063866026699543, 0.0414368212223053, 0.05429811403155327, 0.06593842804431915, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14158985018730164, 0.08735613524913788, 0.041096266359090805, 0.07771317660808563, 0.0664159283041954, 0.06084612384438515, 0.07890869677066803, 0.06467185169458389, 0.041567541658878326, 0.05314023420214653, 0.060331959277391434, 0.049903471022844315, 0.06147686019539833, 0.06152055785059929, 0.053461313247680664, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11277392506599426, 0.05140271782875061, 0.03480074182152748, 0.044109929352998734, 0.04927106201648712, 0.08066023141145706, 0.09371130168437958, 0.07634277641773224, 0.03460073471069336, 0.0555649995803833, 0.09000370651483536, 0.027950923889875412, 0.058086883276700974, 0.09759874641895294, 0.058530546724796295, 0.03459078073501587, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19136789441108704, 0.04588800296187401, 0.037184182554483414, 0.04478847235441208, 0.058455415070056915, 0.0516231432557106, 0.08451797068119049, 0.056617289781570435, 0.04477924853563309, 0.04681586101651192, 0.05177522450685501, 0.034743115305900574, 0.04365590214729309, 0.05257752537727356, 0.045719120651483536, 0.05523938685655594, 0.05425224080681801, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.178098663687706, 0.03651316463947296, 0.035378020256757736, 0.04660060256719589, 0.056067463010549545, 0.05542533099651337, 0.06580902636051178, 0.058695923537015915, 0.04460207000374794, 0.03845333680510521, 0.05522071197628975, 0.02812928520143032, 0.038876280188560486, 0.056352097541093826, 0.04307202994823456, 0.048592109233140945, 0.05801301822066307, 0.056100811809301376, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11277126520872116, 0.04416341334581375, 0.01679084822535515, 0.03922899439930916, 0.030570238828659058, 0.08429381251335144, 0.05829833820462227, 0.07985574752092361, 0.027740228921175003, 0.024607932195067406, 0.0856802761554718, 0.01213553361594677, 0.021223612129688263, 0.08979952335357666, 0.02439562790095806, 0.018000086769461632, 0.0979786366224289, 0.08564859628677368, 0.04681731015443802, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15648671984672546, 0.043593376874923706, 0.029469136148691177, 0.02338278852403164, 0.03989394009113312, 0.06601448357105255, 0.07239893823862076, 0.049598775804042816, 0.025720087811350822, 0.028729889541864395, 0.07268749177455902, 0.020104380324482918, 0.038949914276599884, 0.0779348835349083, 0.031850170344114304, 0.029572613537311554, 0.08162925392389297, 0.04983029142022133, 0.04757083207368851, 0.014582027681171894, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.16675154864788055, 0.040216755121946335, 0.031946271657943726, 0.03915555030107498, 0.04867977276444435, 0.043519362807273865, 0.07027734071016312, 0.04676978662610054, 0.03737045079469681, 0.03785951808094978, 0.04185349866747856, 0.029438791796565056, 0.03496702387928963, 0.04187152162194252, 0.036659594625234604, 0.04418949410319328, 0.04244700446724892, 0.04308999702334404, 0.055408261716365814, 0.02402997761964798, 0.04349849000573158, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.21311703324317932, 0.03452147915959358, 0.026512037962675095, 0.037284668534994125, 0.04200230911374092, 0.05378075689077377, 0.06410719454288483, 0.040195900946855545, 0.029513953253626823, 0.030562065541744232, 0.048140499740839005, 0.019645733758807182, 0.023855963721871376, 0.04712359979748726, 0.027805231511592865, 0.025632821023464203, 0.04690323397517204, 0.042114999145269394, 0.04245338588953018, 0.016209973022341728, 0.04720157012343407, 0.041315577924251556, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.16099101305007935, 0.03149211406707764, 0.025505999103188515, 0.03203628212213516, 0.038119371980428696, 0.05386463925242424, 0.05289200320839882, 0.06297367811203003, 0.03872942551970482, 0.03070300817489624, 0.04861500486731529, 0.016964146867394447, 0.020984871312975883, 0.04743487387895584, 0.026178555563092232, 0.032817039638757706, 0.048741020262241364, 0.03702303394675255, 0.05028446391224861, 0.014425938948988914, 0.04926920309662819, 0.02748514525592327, 0.052469104528427124, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13924191892147064, 0.04732196778059006, 0.022920068353414536, 0.037826746702194214, 0.03733159974217415, 0.056281980127096176, 0.06773107498884201, 0.0638374611735344, 0.028824560344219208, 0.028368612751364708, 0.05008407309651375, 0.014775178395211697, 0.021058905869722366, 0.04819713160395622, 0.017685124650597572, 0.01545622292906046, 0.0484280101954937, 0.037099532783031464, 0.042473260313272476, 0.01842682436108589, 0.04861331358551979, 0.022673960775136948, 0.05998954549431801, 0.02535295858979225, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.195623517036438, 0.03233327716588974, 0.022874081507325172, 0.038390763103961945, 0.03449878469109535, 0.04616905376315117, 0.055537495762109756, 0.03960668295621872, 0.02309839427471161, 0.024249285459518433, 0.04043305665254593, 0.016078554093837738, 0.021624945104122162, 0.03903055936098099, 0.026520881801843643, 0.028113437816500664, 0.03823632746934891, 0.035426102578639984, 0.04174575209617615, 0.02109713852405548, 0.03767955303192139, 0.037286918610334396, 0.04423920437693596, 0.022921495139598846, 0.037184685468673706, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0581965334713459, 0.038897398859262466, 0.02600524015724659, 0.02356790192425251, 0.022979989647865295, 0.053841520100831985, 0.058061059564352036, 0.06858771294355392, 0.037459827959537506, 0.029159093275666237, 0.05474674701690674, 0.013592950068414211, 0.027647046372294426, 0.05575403571128845, 0.030779868364334106, 0.01613735221326351, 0.05745428055524826, 0.03968728333711624, 0.03406553715467453, 0.013110978528857231, 0.059776563197374344, 0.016043761745095253, 0.029699290171265602, 0.020566539838910103, 0.08347290754318237, 0.03070860728621483, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15561480820178986, 0.030323194339871407, 0.02106756716966629, 0.038600414991378784, 0.025089262053370476, 0.04719521105289459, 0.04255649819970131, 0.051662590354681015, 0.030368102714419365, 0.020133620128035545, 0.04255696386098862, 0.018511781468987465, 0.01867171935737133, 0.0414302833378315, 0.022022131830453873, 0.02106102742254734, 0.0416661836206913, 0.027249205857515335, 0.04048159345984459, 0.015536420047283173, 0.04219621419906616, 0.017007341608405113, 0.06220947951078415, 0.015703151002526283, 0.05443187430500984, 0.04092409461736679, 0.015729239210486412, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1521887332201004, 0.03232048079371452, 0.026277603581547737, 0.03215136379003525, 0.03708053380250931, 0.03843334689736366, 0.042185209691524506, 0.034409213811159134, 0.029296185821294785, 0.02475224994122982, 0.03461232781410217, 0.021795086562633514, 0.023419681936502457, 0.03386644273996353, 0.028580598533153534, 0.03440327197313309, 0.033156976103782654, 0.026710400357842445, 0.04213565215468407, 0.019021350890398026, 0.032785814255476, 0.02683998830616474, 0.03304455429315567, 0.02330264076590538, 0.03390035033226013, 0.03944924846291542, 0.021334253251552582, 0.042546432465314865, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10210391879081726, 0.048305340111255646, 0.027957914397120476, 0.028134863823652267, 0.04368893802165985, 0.03765960410237312, 0.04012182354927063, 0.03589581325650215, 0.024491794407367706, 0.019768042489886284, 0.03467409312725067, 0.015431461855769157, 0.032616548240184784, 0.03414339944720268, 0.027461368590593338, 0.020323220640420914, 0.03445841744542122, 0.015827562659978867, 0.030705846846103668, 0.020360933616757393, 0.03558044880628586, 0.034790296107530594, 0.03518148139119148, 0.028572604060173035, 0.04333385452628136, 0.03896389529109001, 0.017587700858712196, 0.04553375393152237, 0.04632510989904404, 0.0, 0.0, 0.0, 0.0], [0.1184174120426178, 0.026689782738685608, 0.02854764647781849, 0.03516455739736557, 0.03519093617796898, 0.030704878270626068, 0.042546890676021576, 0.023103507235646248, 0.025463124737143517, 0.029678549617528915, 0.028395220637321472, 0.019178256392478943, 0.02383357845246792, 0.027640020474791527, 0.03381752222776413, 0.0396401509642601, 0.027574418112635612, 0.021856164559721947, 0.03966765105724335, 0.03941834345459938, 0.02711562067270279, 0.030498063191771507, 0.034932829439640045, 0.0323205292224884, 0.027840541675686836, 0.03877578675746918, 0.021124795079231262, 0.03861142694950104, 0.032771822065114975, 0.019479982554912567, 0.0, 0.0, 0.0], [0.05433792620897293, 0.019247835502028465, 0.011686461977660656, 0.014118609949946404, 0.01390978042036295, 0.05736807361245155, 0.0345463827252388, 0.046650584787130356, 0.013026447966694832, 0.013271295465528965, 0.05952775105834007, 0.009063147008419037, 0.017364196479320526, 0.061355650424957275, 0.01263465266674757, 0.015145537443459034, 0.06425532698631287, 0.046270616352558136, 0.01955636776983738, 0.012543462216854095, 0.06913558393716812, 0.034823667258024216, 0.030005954205989838, 0.012261823751032352, 0.07985591888427734, 0.018996428698301315, 0.010722929611802101, 0.0865454226732254, 0.017174094915390015, 0.022144164890050888, 0.022453978657722473, 0.0, 0.0], [0.05849646031856537, 0.03437737002968788, 0.013603095896542072, 0.02417043223977089, 0.01900586113333702, 0.0654640942811966, 0.030622966587543488, 0.03573206812143326, 0.01885945349931717, 0.012241484597325325, 0.06282190978527069, 0.011835752986371517, 0.009552359580993652, 0.06431794911623001, 0.02158958464860916, 0.010912974365055561, 0.0666591003537178, 0.02399745211005211, 0.026596758514642715, 0.006153556052595377, 0.06984329968690872, 0.03723869100213051, 0.030046707019209862, 0.021686814725399017, 0.043640609830617905, 0.021209606900811195, 0.013490457087755203, 0.04632722586393356, 0.020434511825442314, 0.013362264260649681, 0.018568990752100945, 0.04714024439454079, 0.0], [0.1352868676185608, 0.02973337098956108, 0.02611132338643074, 0.03753575682640076, 0.038881585001945496, 0.0331667996942997, 0.04824390634894371, 0.03046831116080284, 0.026038208976387978, 0.024123696610331535, 0.027892732992768288, 0.02270994521677494, 0.024185286834836006, 0.02611454948782921, 0.02320966310799122, 0.025175290182232857, 0.024614792317152023, 0.02112315408885479, 0.02968505024909973, 0.01816747523844242, 0.023064401000738144, 0.02059807814657688, 0.034411393105983734, 0.021083395928144455, 0.02253883145749569, 0.0322815477848053, 0.013987251557409763, 0.02414492331445217, 0.02700231224298477, 0.012929446063935757, 0.03616539016366005, 0.02684023790061474, 0.03248502314090729]]]}\n",
        "    )\n",
        "    </script>"
       ],
       "text/plain": [
-       "<circuitsvis.utils.render.RenderedHTML at 0x7f1034184810>"
+       "<circuitsvis.utils.render.RenderedHTML at 0x7f026fd86f10>"
       ]
      },
      "execution_count": 13,
@@ -521,10 +521,10 @@
    "execution_count": 14,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:26.326833Z",
-     "iopub.status.busy": "2024-12-14T01:01:26.326375Z",
-     "iopub.status.idle": "2024-12-14T01:01:26.344132Z",
-     "shell.execute_reply": "2024-12-14T01:01:26.343488Z"
+     "iopub.execute_input": "2024-12-31T02:20:43.678561Z",
+     "iopub.status.busy": "2024-12-31T02:20:43.678246Z",
+     "iopub.status.idle": "2024-12-31T02:20:43.695529Z",
+     "shell.execute_reply": "2024-12-31T02:20:43.695017Z"
     }
    },
    "outputs": [],
@@ -579,10 +579,10 @@
    "execution_count": 15,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:26.346946Z",
-     "iopub.status.busy": "2024-12-14T01:01:26.346593Z",
-     "iopub.status.idle": "2024-12-14T01:01:26.595536Z",
-     "shell.execute_reply": "2024-12-14T01:01:26.594923Z"
+     "iopub.execute_input": "2024-12-31T02:20:43.698096Z",
+     "iopub.status.busy": "2024-12-31T02:20:43.697716Z",
+     "iopub.status.idle": "2024-12-31T02:20:43.947211Z",
+     "shell.execute_reply": "2024-12-31T02:20:43.946583Z"
     }
    },
    "outputs": [
@@ -665,10 +665,10 @@
    "execution_count": 16,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:26.598076Z",
-     "iopub.status.busy": "2024-12-14T01:01:26.597740Z",
-     "iopub.status.idle": "2024-12-14T01:01:26.804705Z",
-     "shell.execute_reply": "2024-12-14T01:01:26.804124Z"
+     "iopub.execute_input": "2024-12-31T02:20:43.949512Z",
+     "iopub.status.busy": "2024-12-31T02:20:43.949339Z",
+     "iopub.status.idle": "2024-12-31T02:20:44.148098Z",
+     "shell.execute_reply": "2024-12-31T02:20:44.147509Z"
     }
    },
    "outputs": [
@@ -721,17 +721,17 @@
    "execution_count": 17,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:26.807213Z",
-     "iopub.status.busy": "2024-12-14T01:01:26.806795Z",
-     "iopub.status.idle": "2024-12-14T01:01:46.682287Z",
-     "shell.execute_reply": "2024-12-14T01:01:46.681658Z"
+     "iopub.execute_input": "2024-12-31T02:20:44.150540Z",
+     "iopub.status.busy": "2024-12-31T02:20:44.150152Z",
+     "iopub.status.idle": "2024-12-31T02:21:03.174534Z",
+     "shell.execute_reply": "2024-12-31T02:21:03.173913Z"
     }
    },
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "ddd91d21a45e49a8a90fd6693fc0fe90",
+       "model_id": "bb02e41e51664a96b3c06abd21683006",
        "version_major": 2,
        "version_minor": 0
       },
@@ -788,10 +788,10 @@
    "execution_count": 18,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:46.684826Z",
-     "iopub.status.busy": "2024-12-14T01:01:46.684383Z",
-     "iopub.status.idle": "2024-12-14T01:01:46.842465Z",
-     "shell.execute_reply": "2024-12-14T01:01:46.841862Z"
+     "iopub.execute_input": "2024-12-31T02:21:03.177037Z",
+     "iopub.status.busy": "2024-12-31T02:21:03.176647Z",
+     "iopub.status.idle": "2024-12-31T02:21:03.328310Z",
+     "shell.execute_reply": "2024-12-31T02:21:03.327647Z"
     }
    },
    "outputs": [
@@ -802,9 +802,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"a86e9b27-7dd3-47fb-9141-10fa1ea8195f\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"a86e9b27-7dd3-47fb-9141-10fa1ea8195f\")) {                    Plotly.newPlot(                        \"a86e9b27-7dd3-47fb-9141-10fa1ea8195f\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"x\":[\"\\u003c|endoftext|\\u003e_0\",\"After_1\",\" John_2\",\" and_3\",\" Mary_4\",\" went_5\",\" to_6\",\" the_7\",\" store_8\",\",_9\",\" Mary_10\",\" gave_11\",\" a_12\",\" bottle_13\",\" of_14\",\" milk_15\",\" to_16\"],\"z\":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9981481432914734,0.001600604155100882,0.0001503795647295192,-0.0003718698862940073,-2.257052983622998e-05,-0.000628303736448288,-0.000515179184731096],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9980564713478088,0.002283838577568531,0.00018205988453701138,-0.0005045737489126623,-0.0002681270125322044,-5.153151141712442e-05,-0.0012816254748031497],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9967373609542847,0.004081866703927517,0.0009732521721161902,4.378139055916108e-05,-0.00015962531324476004,-0.00033638247987255454,-0.0019441929180175066],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9905908107757568,0.019987020641565323,0.0018955166451632977,0.0010141782695427537,-6.75756236887537e-05,0.000911251176148653,-0.0019012272823601961],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9616515636444092,0.08534706383943558,0.0052040028385818005,0.003052052343264222,0.00019688029715325683,0.0011058200616389513,-0.002284654416143894],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.963099479675293,0.0843719094991684,0.00412211287766695,0.0007176341023296118,0.00010319898865418509,0.0010020771296694875,-0.004215250723063946],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9359182119369507,0.11111730337142944,0.007705116644501686,0.0003754050412680954,0.0003647996054496616,0.0013266304740682244,0.01874428242444992],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.7701544761657715,0.03741976246237755,0.0020680588204413652,-8.321183850057423e-05,0.00013447141100186855,0.0017246060306206346,0.44990694522857666],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0965060144662857,0.025925789028406143,0.0019711144268512726,0.00032904025283642113,0.00042367333662696183,0.00188477523624897,0.8994728922843933],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.023322155699133873,0.018537340685725212,0.001587143400683999,0.000526600400917232,0.0002531706413719803,0.0008727724780328572,0.9612765908241272],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.008559261448681355,0.006339463870972395,0.0005807152483612299,-0.000342636980349198,0.00010958944039884955,0.0006486988277174532,0.9495824575424194]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Position: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Position\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Normalized Logit Difference After Patching Residual Stream on the IOI Task\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"2edcf647-342a-478a-992e-622f7caf82dc\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"2edcf647-342a-478a-992e-622f7caf82dc\")) {                    Plotly.newPlot(                        \"2edcf647-342a-478a-992e-622f7caf82dc\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"x\":[\"\\u003c|endoftext|\\u003e_0\",\"After_1\",\" John_2\",\" and_3\",\" Mary_4\",\" went_5\",\" to_6\",\" the_7\",\" store_8\",\",_9\",\" Mary_10\",\" gave_11\",\" a_12\",\" bottle_13\",\" of_14\",\" milk_15\",\" to_16\"],\"z\":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9981481432914734,0.001600604155100882,0.0001503795647295192,-0.0003718698862940073,-2.257052983622998e-05,-0.000628303736448288,-0.000515179184731096],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9980564713478088,0.002283838577568531,0.00018205988453701138,-0.0005045737489126623,-0.0002681270125322044,-5.153151141712442e-05,-0.0012816254748031497],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9967373609542847,0.004081866703927517,0.0009732521721161902,4.378139055916108e-05,-0.00015962531324476004,-0.00033638247987255454,-0.0019441929180175066],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9905908107757568,0.019987020641565323,0.0018955166451632977,0.0010141782695427537,-6.75756236887537e-05,0.000911251176148653,-0.0019012272823601961],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9616515636444092,0.08534706383943558,0.0052040028385818005,0.003052052343264222,0.00019688029715325683,0.0011058200616389513,-0.002284654416143894],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.963099479675293,0.0843719094991684,0.00412211287766695,0.0007176341023296118,0.00010319898865418509,0.0010020771296694875,-0.004215250723063946],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9359182119369507,0.11111730337142944,0.007705116644501686,0.0003754050412680954,0.0003647996054496616,0.0013266304740682244,0.01874428242444992],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.7701544761657715,0.03741976246237755,0.0020680588204413652,-8.321183850057423e-05,0.00013447141100186855,0.0017246060306206346,0.44990694522857666],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0965060144662857,0.025925789028406143,0.0019711144268512726,0.00032904025283642113,0.00042367333662696183,0.00188477523624897,0.8994728922843933],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.023322155699133873,0.018537340685725212,0.001587143400683999,0.000526600400917232,0.0002531706413719803,0.0008727724780328572,0.9612765908241272],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.008559261448681355,0.006339463870972395,0.0005807152483612299,-0.000342636980349198,0.00010958944039884955,0.0006486988277174532,0.9495824575424194]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Position: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Position\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Normalized Logit Difference After Patching Residual Stream on the IOI Task\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('a86e9b27-7dd3-47fb-9141-10fa1ea8195f');\n",
+       "var gd = document.getElementById('2edcf647-342a-478a-992e-622f7caf82dc');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -881,10 +881,10 @@
    "execution_count": 19,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:46.844857Z",
-     "iopub.status.busy": "2024-12-14T01:01:46.844522Z",
-     "iopub.status.idle": "2024-12-14T01:01:48.672019Z",
-     "shell.execute_reply": "2024-12-14T01:01:48.671432Z"
+     "iopub.execute_input": "2024-12-31T02:21:03.330925Z",
+     "iopub.status.busy": "2024-12-31T02:21:03.330740Z",
+     "iopub.status.idle": "2024-12-31T02:21:05.098330Z",
+     "shell.execute_reply": "2024-12-31T02:21:05.097767Z"
     }
    },
    "outputs": [
@@ -895,9 +895,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"d81bb9d3-dd8a-47ab-a254-a6ee2bc9adcc\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"d81bb9d3-dd8a-47ab-a254-a6ee2bc9adcc\")) {                    Plotly.newPlot(                        \"d81bb9d3-dd8a-47ab-a254-a6ee2bc9adcc\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98],\"xaxis\":\"x\",\"y\":[11.934269905090332,14.23181438446045,11.729150772094727,15.214762687683105,13.592933654785156,14.211908340454102,13.205050468444824,13.07355785369873,11.535038948059082,12.773419380187988,13.354753494262695,12.027746200561523,12.879249572753906,11.845925331115723,12.831506729125977,11.792170524597168,12.737360000610352,12.617140769958496,12.619315147399902,12.312422752380371,11.479436874389648,12.757272720336914,11.545267105102539,12.173948287963867,11.990314483642578,11.905129432678223,11.678251266479492,11.099139213562012,10.786980628967285,12.338418960571289,11.62226676940918,11.612360000610352,10.657169342041016,11.615835189819336,11.122636795043945,10.654256820678711,11.861252784729004,11.142145156860352,11.718962669372559,11.342132568359375,11.637442588806152,11.20296859741211,12.066685676574707,10.65315055847168,11.302213668823242,11.483949661254883,10.655682563781738,11.83389949798584,11.766695976257324,11.974489212036133,3.564096450805664,1.2277376651763916,0.9820038676261902,1.200751543045044,0.4091821312904358,0.5843209624290466,0.3591119647026062,0.43036898970603943,0.24223056435585022,0.44049954414367676,0.41071295738220215,0.21330496668815613,0.15890493988990784,0.31460171937942505,0.08366963267326355,0.2029363214969635,0.17958413064479828,0.1480543464422226,0.034813292324543,0.06144014745950699,0.09302718937397003,0.06429728120565414,0.4040629267692566,0.12488675117492676,0.08299500495195389,0.02960968390107155,0.01776202768087387,0.03346758708357811,0.10293184220790863,0.0828002542257309,0.008384309709072113,0.031925592571496964,0.1624717116355896,0.05249609425663948,0.02139073610305786,0.03643417730927467,0.07777489721775055,0.1070786863565445,0.03805286064743996,0.11206188052892685,0.1607268750667572,0.13429032266139984,0.16408373415470123,0.05713384598493576,0.12058018147945404,0.09626038372516632,0.04750080779194832,0.1064792275428772,0.15790846943855286],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"}},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Loss by position on random repeated tokens\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"9fc76eb5-5519-441b-804f-b9d3f284b6b4\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"9fc76eb5-5519-441b-804f-b9d3f284b6b4\")) {                    Plotly.newPlot(                        \"9fc76eb5-5519-441b-804f-b9d3f284b6b4\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98],\"xaxis\":\"x\",\"y\":[11.293755531311035,13.661523818969727,14.26708984375,13.05138874053955,14.200154304504395,13.794110298156738,13.165082931518555,12.725430488586426,12.198771476745605,13.159016609191895,11.78315258026123,11.827463150024414,12.019637107849121,12.807088851928711,12.95734691619873,12.7166109085083,11.805150032043457,11.203890800476074,13.748982429504395,11.939554214477539,12.111616134643555,11.838865280151367,11.640402793884277,11.894838333129883,11.969301223754883,10.904563903808594,11.69995403289795,11.886842727661133,11.184012413024902,11.828400611877441,11.488662719726562,11.595769882202148,11.27686595916748,12.046403884887695,11.61715030670166,11.7785062789917,12.143930435180664,11.37479019165039,11.056105613708496,11.812862396240234,11.769815444946289,11.787055969238281,11.405266761779785,11.02073860168457,12.023134231567383,11.26836109161377,10.82592487335205,11.290939331054688,11.096704483032227,11.10577392578125,3.174548387527466,1.6711616516113281,0.613044261932373,0.33394116163253784,0.4114641547203064,0.5941733717918396,0.30370816588401794,0.14971032738685608,0.16561320424079895,0.1323113888502121,0.19412001967430115,0.2268151342868805,0.08987182378768921,0.03984134644269943,0.09262757748365402,0.1882668137550354,0.16898849606513977,0.15458349883556366,0.09763779491186142,0.12957245111465454,0.2606273293495178,0.03914522007107735,0.049064647406339645,0.0566009059548378,0.012886052019894123,0.031596552580595016,0.04321982339024544,0.07121316343545914,0.0856885313987732,0.023817967623472214,0.048696257174015045,0.0647251158952713,0.07937098294496536,0.1875138133764267,0.05469178408384323,0.04853184148669243,0.04821369796991348,0.10546024888753891,0.16193492710590363,0.02764531411230564,0.1413104087114334,0.04153662547469139,0.11765837669372559,0.04498446732759476,0.2020321786403656,0.06594666093587875,0.06909110397100449,0.08714185655117035,0.10666583478450775],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"}},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Loss by position on random repeated tokens\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('d81bb9d3-dd8a-47ab-a254-a6ee2bc9adcc');\n",
+       "var gd = document.getElementById('9fc76eb5-5519-441b-804f-b9d3f284b6b4');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -963,10 +963,10 @@
    "execution_count": 20,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:48.674292Z",
-     "iopub.status.busy": "2024-12-14T01:01:48.673943Z",
-     "iopub.status.idle": "2024-12-14T01:01:49.886399Z",
-     "shell.execute_reply": "2024-12-14T01:01:49.885883Z"
+     "iopub.execute_input": "2024-12-31T02:21:05.100374Z",
+     "iopub.status.busy": "2024-12-31T02:21:05.100199Z",
+     "iopub.status.idle": "2024-12-31T02:21:06.275048Z",
+     "shell.execute_reply": "2024-12-31T02:21:06.274508Z"
     }
    },
    "outputs": [
@@ -977,9 +977,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"b2a3ed8a-ba81-4aac-a1a1-60e16774f655\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"b2a3ed8a-ba81-4aac-a1a1-60e16774f655\")) {                    Plotly.newPlot(                        \"b2a3ed8a-ba81-4aac-a1a1-60e16774f655\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.010606949217617512,0.00013902947830501944,0.01081207673996687,5.408500101111713e-07,0.00022116937907412648,7.771956006763503e-05,0.00958997942507267,0.0009253817261196673,0.009193798527121544,0.009053227491676807,0.006911635398864746,0.014761571772396564],[0.0008132570656016469,0.0004268861666787416,0.002124645747244358,0.01425299420952797,0.005162382964044809,0.010268471203744411,0.015934234485030174,0.013339280150830746,0.012906215153634548,0.016355415806174278,0.006486278492957354,0.0005485203000716865],[0.004910458344966173,0.019308894872665405,0.0012424595188349485,0.0010258526308462024,0.013039778918027878,0.0015818236861377954,0.00429101288318634,0.008304944261908531,0.003105038544163108,0.000971563218627125,0.0007218025857582688,0.010301434434950352],[0.015241926535964012,0.006064379122108221,0.00234038638882339,0.01002125907689333,0.021940935403108597,0.01423431932926178,0.0017911610193550587,0.0017028121510520577,0.005022364668548107,0.012421459890902042,0.00839681550860405,0.005942116491496563],[0.015156026929616928,0.013609741814434528,0.01541218627244234,0.00972918700426817,0.018237268552184105,0.013402285054326057,0.009189526550471783,0.0018425184534862638,0.015492293052375317,0.013836968690156937,0.0181787870824337,3.1250738885546525e-08],[0.42406418919563293,0.9055006504058838,0.015690578147768974,0.01017544511705637,0.01198639441281557,0.9278296828269958,0.012438608333468437,0.018412206321954727,0.027197834104299545,0.028249241411685944,0.019754743203520775,0.018243789672851562],[0.012461011298000813,0.01634187251329422,0.019040238112211227,0.015691598877310753,0.02333749644458294,0.012633594684302807,0.029116196557879448,0.012918342836201191,0.009929743595421314,0.92023104429245,0.035106346011161804,0.015322730876505375],[0.011738991364836693,0.17485056817531586,0.8415674567222595,0.01858351193368435,0.018508559092879295,0.018142390996217728,0.04514370858669281,0.08067239820957184,0.017961328849196434,0.019256191328167915,0.9187174439430237,0.052430037409067154],[0.01744176633656025,0.3839356303215027,0.01593034528195858,0.0451587438583374,0.018434638157486916,0.01456737332046032,0.14198516309261322,0.014199082739651203,0.03386015444993973,0.0334649458527565,0.06564274430274963,0.02554652839899063],[0.2423967868089676,0.19136165082454681,0.1043233796954155,0.012038059532642365,0.0845528319478035,0.027191946282982826,0.4313036799430847,0.028380122035741806,0.04645952582359314,0.4677841365337372,0.017513224855065346,0.03886174410581589],[0.3296402394771576,0.48611578345298767,0.0376255176961422,0.14191953837871552,0.06014999374747276,0.01589331030845642,0.28858682513237,0.479020357131958,0.05016819015145302,0.015279495157301426,0.14848238229751587,0.25642627477645874],[0.01643357053399086,0.05329973250627518,0.035110954195261,0.011042965576052666,0.03575966879725456,0.10564529150724411,0.048889338970184326,0.06571993976831436,0.009506475180387497,0.30028679966926575,0.3763805329799652,0.025962304323911667]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Induction Score by Head\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"ecf684dd-de93-401f-8da4-35a69a99f28f\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"ecf684dd-de93-401f-8da4-35a69a99f28f\")) {                    Plotly.newPlot(                        \"ecf684dd-de93-401f-8da4-35a69a99f28f\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.010750330053269863,0.00012081558088539168,0.010657578706741333,7.758595756968134e-07,0.00018270811415277421,0.0001077339329640381,0.009928593412041664,0.0006260719965212047,0.009389315731823444,0.009186347015202045,0.00702526094391942,0.014964522793889046],[0.0007892780122347176,0.0003822337312158197,0.0021375513169914484,0.014263155870139599,0.005154898855835199,0.010422330349683762,0.015344158746302128,0.012839337810873985,0.013111173175275326,0.01605435460805893,0.006542571354657412,0.0005330032436177135],[0.004155242349952459,0.017997099086642265,0.0025227037258446217,0.0013244586298242211,0.014491848647594452,0.002136210212484002,0.003679797286167741,0.008564706891775131,0.003503501648083329,0.001399003784172237,0.0006941384635865688,0.010122609324753284],[0.015284194611012936,0.007457489613443613,0.0030496998224407434,0.01488673035055399,0.02102597802877426,0.011635568924248219,0.002769281156361103,0.0009841499850153923,0.005578458774834871,0.014012619853019714,0.008734799921512604,0.007212471682578325],[0.016826970502734184,0.01410447433590889,0.014252868480980396,0.008973448537290096,0.018241047859191895,0.014347823336720467,0.008000489324331284,0.002079280326142907,0.015254753641784191,0.01450068037956953,0.018436405807733536,8.373506332759462e-10],[0.4491550922393799,0.9050912857055664,0.015386047773063183,0.00875406339764595,0.01097724586725235,0.9397631883621216,0.011584484949707985,0.01816929504275322,0.028745127841830254,0.027196679264307022,0.017204662784934044,0.017268218100070953],[0.010569052770733833,0.016979962587356567,0.01757051981985569,0.013518791645765305,0.021789412945508957,0.01171425823122263,0.02984258159995079,0.010995764285326004,0.00976125244051218,0.9289324283599854,0.03460867702960968,0.014708924107253551],[0.014089317992329597,0.18321795761585236,0.850297749042511,0.018563423305749893,0.0190876517444849,0.01761588081717491,0.04573224112391472,0.08691924065351486,0.01809566281735897,0.01936078630387783,0.9231538772583008,0.05635310709476471],[0.01650477573275566,0.38255754113197327,0.01667136326432228,0.04383406788110733,0.01883339323103428,0.01519932970404625,0.14662882685661316,0.012798622250556946,0.02996883913874626,0.03196403756737709,0.06725890934467316,0.019399145618081093],[0.2549801766872406,0.17897091805934906,0.11086569726467133,0.013412022031843662,0.09373626857995987,0.02757657691836357,0.43764016032218933,0.02954663708806038,0.05084982514381409,0.4542059898376465,0.018278131261467934,0.044380251318216324],[0.35109150409698486,0.4873502254486084,0.04331290349364281,0.1357107311487198,0.06467364728450775,0.016985448077321053,0.3039875626564026,0.46872273087501526,0.050012968480587006,0.01698106899857521,0.16451457142829895,0.2619016468524933],[0.017976440489292145,0.0546535849571228,0.03751886636018753,0.011606932617723942,0.0373089537024498,0.10333973169326782,0.054286956787109375,0.06869052350521088,0.009068110026419163,0.3063192069530487,0.4115610718727112,0.023157836869359016]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Induction Score by Head\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('b2a3ed8a-ba81-4aac-a1a1-60e16774f655');\n",
+       "var gd = document.getElementById('ecf684dd-de93-401f-8da4-35a69a99f28f');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -1054,28 +1054,28 @@
    "execution_count": 21,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:49.888604Z",
-     "iopub.status.busy": "2024-12-14T01:01:49.888277Z",
-     "iopub.status.idle": "2024-12-14T01:01:50.092655Z",
-     "shell.execute_reply": "2024-12-14T01:01:50.091997Z"
+     "iopub.execute_input": "2024-12-31T02:21:06.277258Z",
+     "iopub.status.busy": "2024-12-31T02:21:06.276890Z",
+     "iopub.status.idle": "2024-12-31T02:21:06.477972Z",
+     "shell.execute_reply": "2024-12-31T02:21:06.477418Z"
     }
    },
    "outputs": [
     {
      "data": {
       "text/html": [
-       "<div id=\"circuits-vis-61a73e43-0cbc\" style=\"margin: 15px 0;\"/>\n",
+       "<div id=\"circuits-vis-fe1e1e85-ccb2\" style=\"margin: 15px 0;\"/>\n",
        "    <script crossorigin type=\"module\">\n",
        "    import { render, AttentionPatterns } from \"https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js\";\n",
        "    render(\n",
-       "      \"circuits-vis-61a73e43-0cbc\",\n",
+       "      \"circuits-vis-fe1e1e85-ccb2\",\n",
        "      AttentionPatterns,\n",
        "      {\"tokens\": [\"use\", \" advice\", \" Social\", \"\\u00f6\", \"\\u00b7\", \" fought\", \" Le\", \" allegedly\", \" NO\", \"alth\", \"car\", \" prepared\", \"new\", \"rant\", \"roll\", \" hours\", \" published\", \"66\", \"ension\", \" 44\", \"use\", \" advice\", \" Social\", \"\\u00f6\", \"\\u00b7\", \" fought\", \" Le\", \" allegedly\", \" NO\", \"alth\", \"car\", \" prepared\", \"new\", \"rant\", \"roll\", \" hours\", \" published\", \"66\", \"ension\", \" 44\"], \"attention\": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9737270474433899, 0.02627299167215824, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9820428490638733, 0.017020218074321747, 0.0009368443279527128, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9895542860031128, 0.008665801025927067, 0.00041197543032467365, 0.0013679895782843232, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8543054461479187, 0.07801821082830429, 0.0008415375486947596, 0.0001359905581921339, 0.06669880449771881, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.937433660030365, 0.03300206735730171, 0.0015577399171888828, 2.535264457037556e-06, 0.0010925547685474157, 0.026911458000540733, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9769211411476135, 0.0038436795584857464, 2.234023304481525e-05, 3.5218730772612616e-05, 0.00518348254263401, 0.012176254764199257, 0.0018179015023633838, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9473504424095154, 0.013174979016184807, 0.0013492192374542356, 1.1802491826529149e-05, 0.0009449502103962004, 0.01131904125213623, 0.018021006137132645, 0.00782855786383152, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9847128391265869, 0.0010781444143503904, 0.0021734496112912893, 5.4822653510200325e-06, 0.0004914247547276318, 0.0013570826267823577, 0.00010185814608121291, 0.0002853855839930475, 0.009794436395168304, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9915198683738708, 0.004483341239392757, 0.00012727164721582085, 0.00016702155699022114, 0.0016301742289215326, 0.0011521608103066683, 0.00032312856637872756, 0.00012646272080019116, 0.00039313736488111317, 7.735046528978273e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8908806443214417, 0.024311941117048264, 1.7341310012852773e-05, 4.157719376962632e-05, 0.0008967609610408545, 0.07334917038679123, 0.0009482790483161807, 0.004280843771994114, 0.00516867870464921, 7.830337381165009e-06, 9.693232277641073e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8170808553695679, 0.1351710557937622, 0.011989916674792767, 1.1421708222769666e-05, 0.0003511959221214056, 0.009450665675103664, 0.01946333795785904, 0.0006557459710165858, 0.0005760999629274011, 2.9927030482213013e-05, 1.6589872757322155e-05, 0.00520310876891017, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9082695841789246, 0.006068154238164425, 0.013871855102479458, 0.0008237077272497118, 0.01190832071006298, 0.015542060136795044, 0.008354832418262959, 0.002078164601698518, 0.0013173240004107356, 0.0021398039534687996, 0.0039441632106900215, 0.0012376613449305296, 0.024444298818707466, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9479591250419617, 0.0021026728209108114, 0.011938490904867649, 0.00012338497617747635, 3.5374921480979538e-06, 0.00014498813834507018, 0.0005875456845387816, 2.5534465748933144e-05, 0.0013608969748020172, 0.0003395687963347882, 0.010076268576085567, 0.015790557488799095, 0.006346334703266621, 0.0032010283321142197, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9393549561500549, 0.006392320618033409, 0.0018427377799525857, 6.1162154452176765e-06, 0.00033587400685064495, 0.002051525516435504, 0.003801509505137801, 0.0012357983505353332, 0.00021948102221358567, 0.0003869360953103751, 5.012243491364643e-05, 0.008153212256729603, 0.026924701407551765, 0.002937993500381708, 0.0063067772425711155, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9339622259140015, 0.0017828107811510563, 0.005864636041224003, 0.0001995021157199517, 7.227134483400732e-05, 0.0014535371446982026, 0.0025924311485141516, 0.00048593789688311517, 0.002229832811281085, 0.00015120484749786556, 0.012292915023863316, 0.005057850386947393, 0.01236859429627657, 0.0039444840513169765, 0.006275098770856857, 0.011266663670539856, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8931334018707275, 0.0015468214405700564, 0.013001682236790657, 7.966378689161502e-06, 5.864375998498872e-05, 0.0008863414404913783, 0.00320208678022027, 3.214761454728432e-05, 0.00018022459698840976, 1.1455733329057693e-05, 7.600126264151186e-05, 0.00042027299059554935, 0.0016126197297126055, 0.028539275750517845, 0.01053555216640234, 0.025432037189602852, 0.021323570981621742, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9847024083137512, 0.000458244961919263, 0.00017220497829839587, 6.160992711556901e-07, 4.782819814863615e-06, 0.0005806140834465623, 0.0004461870121303946, 0.0004120134108234197, 0.0013038699980825186, 0.0003176067257300019, 6.994167779339477e-05, 0.0013941116631031036, 5.583090751315467e-05, 0.0009110373794101179, 0.00019558057829272002, 0.0003960288013331592, 0.0011691722320392728, 0.007409730460494757, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8922597765922546, 0.01028366107493639, 0.007569293025881052, 0.015225829556584358, 0.0006035296246409416, 0.0014377726474776864, 0.018397418782114983, 0.00018186635861638933, 0.002113566268235445, 3.803668369073421e-05, 0.009962561540305614, 0.003998206928372383, 0.0012666863622143865, 0.0021862699650228024, 0.0032670684158802032, 0.0015871950890868902, 0.01913381926715374, 0.008779395371675491, 0.0017080748220905662, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5637838244438171, 2.204169322794769e-05, 0.0003808305482380092, 1.393800062032824e-07, 4.306913226059805e-08, 0.00012883225281257182, 5.202722968533635e-05, 4.098194494872587e-06, 0.00043821518192999065, 1.0102487067342736e-05, 2.0490055248956196e-05, 0.00021747536084149033, 2.5250110411434434e-05, 2.129377389792353e-05, 0.0022071911953389645, 5.892770423088223e-05, 0.002418374642729759, 0.003277554176747799, 0.42604929208755493, 0.0008841048693284392, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14635005593299866, 0.45109108090400696, 0.027205264195799828, 0.0030082776211202145, 0.0007913765148259699, 0.0800931379199028, 0.005927647929638624, 0.0006846337346360087, 0.0021268464624881744, 0.0027747468557208776, 0.00023907337163109332, 0.002550546545535326, 0.0054934462532401085, 0.01583220437169075, 0.0003450022195465863, 0.0005726688541471958, 0.002175167202949524, 0.039043255150318146, 0.16982686519622803, 0.04120763763785362, 0.0026610149070620537, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15883015096187592, 0.09413880854845047, 0.6926899552345276, 4.776390778715722e-05, 8.085349691100419e-06, 0.00935556460171938, 0.0008445517742075026, 2.443764515192015e-06, 0.00013773982936982065, 1.1189692941115936e-06, 4.677353445003973e-06, 0.0003472122480161488, 0.002631494775414467, 0.0004504133830778301, 0.006463215220719576, 0.0005723336944356561, 0.0012668337440118194, 0.006402322091162205, 0.0018092951504513621, 0.006555411033332348, 0.00037914939457550645, 0.017061391845345497, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05072391405701637, 0.0043561081402003765, 0.00013167233555577695, 0.9396469593048096, 0.0005500844563357532, 0.002771171974018216, 1.4556275345967151e-05, 5.0173721319879405e-06, 1.5498681023018435e-05, 7.020712899930004e-08, 8.694933967490215e-06, 3.654160536825657e-05, 3.607992539400584e-06, 2.5941648345906287e-05, 7.590860604977934e-06, 7.10092763256398e-07, 4.6298235247377306e-05, 7.143522088881582e-05, 0.00012089123629266396, 0.0005609994404949248, 1.3380947166297119e-05, 0.0007342157769016922, 0.0001547116116853431, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.044663090258836746, 0.0028326627798378468, 7.648707105545327e-05, 0.00015513667312916368, 0.7502228021621704, 0.19195663928985596, 9.640512871555984e-05, 0.00016210127796512097, 0.00012769461318384856, 1.1226569768041372e-05, 8.733285540074576e-06, 0.0002813320606946945, 5.2078037697356194e-05, 0.008386553265154362, 4.340289706306066e-06, 6.482724711531773e-05, 3.802955325227231e-05, 7.603636913700029e-05, 0.00012636324390769005, 9.227949340129271e-05, 4.0301659964825376e-07, 0.00011281677143415436, 2.522385329939425e-06, 0.0004493836604524404, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.013606224209070206, 0.006971816532313824, 3.1886105716694146e-05, 1.8455255030858098e-06, 0.002301084576174617, 0.9711790084838867, 0.0003632458101492375, 8.45977701828815e-05, 0.0001061156508512795, 4.5056845010549296e-07, 2.987579534874385e-07, 0.0008592646918259561, 8.844918920658529e-05, 0.00034812072408385575, 9.285179203288862e-07, 3.160578853567131e-05, 1.2802144738088828e-05, 5.8033951063407585e-05, 0.00010517514601815492, 6.438296259148046e-05, 1.8867377775677596e-06, 0.0009238758939318359, 4.68101461592596e-06, 8.99309179658303e-06, 0.0028452184051275253, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01961127668619156, 0.00438615120947361, 6.198724440764636e-05, 4.88576041846045e-08, 4.252470171195455e-05, 0.0036122461315244436, 0.9598402380943298, 0.005622219294309616, 0.0023445591796189547, 5.173985186956997e-07, 1.9621675164671615e-06, 0.00165482924785465, 0.0005915391957387328, 0.001169584458693862, 5.784227596450364e-06, 8.119036647258326e-05, 4.500302384258248e-05, 0.00018497802375350147, 5.086949386168271e-05, 0.00011132832878502086, 6.848460998298833e-06, 3.8539848901564255e-05, 4.853054633713327e-06, 1.276350758416811e-07, 3.953097802877892e-06, 0.0005269531393423676, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.008729088120162487, 0.0001781070022843778, 2.8806286422877747e-07, 4.138263136610476e-07, 6.864999886602163e-05, 0.0009331137989647686, 0.0001488832349423319, 0.984452486038208, 0.004786369390785694, 0.0001132894612965174, 7.255491141222592e-07, 7.424689101753756e-05, 1.4996417121437844e-05, 0.00019790598889812827, 2.995051318066544e-07, 4.872014415013837e-06, 5.296082690620096e-06, 4.536818778433371e-06, 0.00011268883827142417, 3.4170998333138414e-06, 8.272462764580268e-06, 1.23798881759285e-05, 3.665505587946427e-08, 9.0251188566981e-07, 1.6053050785558298e-05, 0.00011404338874854147, 1.8653627193998545e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04082155600190163, 0.003618426388129592, 4.821046968572773e-05, 1.9615674773376668e-07, 5.3802777983946726e-05, 0.0021802401170134544, 0.0033272523432970047, 0.001878200564533472, 0.9307873249053955, 0.00481536565348506, 1.443025394110009e-05, 0.0028287956956773996, 0.00010464123624842614, 0.0029309086967259645, 0.0009514776756986976, 6.522179319290444e-05, 0.00029541170806623995, 0.00012317505024839193, 0.0015512284589931369, 0.0005327722756192088, 0.0002208336372859776, 0.00044327162322588265, 1.9391492969589308e-05, 5.825700100103859e-07, 1.6632819097139873e-05, 0.0004773235123138875, 0.0011258182348683476, 0.0007676867535337806, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13800986111164093, 0.000338585494318977, 0.00015972793335095048, 3.422598027214008e-08, 1.2696453268290497e-05, 0.00024196873710025102, 3.82194593839813e-05, 3.751534677576274e-05, 0.004743305500596762, 0.8406771421432495, 0.00018875481327995658, 0.00015232685836963356, 1.250763853022363e-05, 0.0002236300497315824, 0.0001328253565588966, 6.470848165918142e-05, 0.00013950491847936064, 9.763600246515125e-05, 0.0004387965309433639, 4.486108446144499e-05, 6.358909013215452e-05, 0.00034256128128618, 5.2226907428121194e-05, 5.293519507176825e-07, 4.435913979250472e-06, 6.543510244227946e-05, 2.9145055577828316e-06, 3.8726907405362e-06, 0.013709785416722298, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0077987853437662125, 0.00022834629635326564, 4.638521033939469e-07, 6.367737910295546e-07, 1.5816805898793973e-05, 4.74391745228786e-05, 5.2391105782589875e-06, 7.306586212507682e-06, 1.252215042768512e-05, 3.330249569444277e-07, 0.9909055233001709, 0.00045220632455311716, 5.365398919821018e-06, 7.535894837928936e-05, 8.799969691608567e-06, 7.895076123531908e-06, 0.00023918210354167968, 2.256911329823197e-06, 7.641898992005736e-05, 5.514193617273122e-05, 4.378300673124613e-06, 2.181093987019267e-05, 4.435156242266203e-08, 1.2584397381942836e-06, 2.069822357952944e-06, 1.2951757526025176e-05, 3.952872305035271e-07, 1.1202362202311633e-06, 7.981805538292974e-06, 2.9978705242683645e-06, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04228433594107628, 0.01130534615367651, 7.640720696144854e-07, 2.2586132217838895e-06, 9.786387818166986e-05, 0.03826921060681343, 0.00023640895960852504, 0.001455514575354755, 0.0029497891664505005, 1.2863508800364798e-06, 2.8271693736314774e-05, 0.8946183323860168, 0.0005235012504272163, 0.0011799612548202276, 0.0009106284123845398, 0.0003604615631047636, 0.00020461656094994396, 2.910211514972616e-05, 0.0013425340875983238, 0.0003398252301849425, 0.00033254953450523317, 8.430417801719159e-05, 7.937666879342942e-08, 5.80987489229301e-06, 4.91085984322126e-06, 0.002387275919318199, 2.696716364880558e-05, 0.00020931514154653996, 0.0007688859477639198, 5.801381576020503e-06, 3.4147800761274993e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0061029596254229546, 0.016550516709685326, 4.6485445636790246e-05, 8.233629245069096e-08, 7.398499292321503e-06, 0.0005482283304445446, 0.0005931411869823933, 1.5738387446617708e-05, 3.503288098727353e-05, 3.3504278462714865e-07, 1.8657722478110372e-07, 0.00046066963113844395, 0.8416223526000977, 0.12311305105686188, 0.006359235383570194, 0.0026995246298611164, 0.00016676251834724098, 0.001213077805005014, 0.00013198796659708023, 8.468224405078217e-05, 4.615935722540598e-06, 6.994968862272799e-05, 4.916098987450823e-06, 3.4761444567266153e-07, 3.7119539797458856e-07, 6.107360968599096e-05, 6.728529115207493e-05, 2.2758390514354687e-06, 2.0601644791895524e-05, 7.198171942945919e-07, 2.78813168108627e-08, 1.6529560525668785e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04041679948568344, 0.001346431439742446, 0.0002881708787754178, 4.343534783401992e-06, 0.0006837420514784753, 0.0021492524538189173, 0.0012713877949863672, 0.00021145616483408958, 0.0001491332659497857, 2.3681526727159508e-05, 9.255093027604744e-05, 7.460857159458101e-05, 0.0030381479300558567, 0.9153464436531067, 0.002063496969640255, 0.002670764224603772, 0.0006971358670853078, 0.022915009409189224, 0.0016386171337217093, 0.0002943731960840523, 8.301850357383955e-06, 0.0001194091746583581, 3.610825660871342e-05, 1.2593762221513316e-05, 0.00038307331851683557, 0.0011952053755521774, 0.0001844509970396757, 7.125815318431705e-05, 0.00012850550410803407, 6.062198372092098e-05, 4.517644993029535e-05, 6.895749606883328e-07, 0.002379077021032572, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.03775661438703537, 0.0016132340533658862, 0.0003177856560796499, 8.17116472262569e-07, 4.6315386725837016e-07, 7.222060230560601e-05, 0.00012490959488786757, 4.99756515637273e-06, 0.00030667142709717155, 5.749376668973127e-06, 0.00024661491625010967, 0.003777766600251198, 0.0013358162250369787, 0.002113183494657278, 0.8997159600257874, 0.04417850822210312, 0.00033094940590672195, 0.0013849412789568305, 0.00039355995249934494, 0.002321068197488785, 0.000464222306618467, 0.00019271834753453732, 0.0005703889764845371, 5.927758593315957e-06, 2.4687206945372964e-08, 3.1083311569091165e-06, 7.165991064539412e-06, 2.9240266030683415e-07, 7.984187686815858e-05, 1.2546481229946949e-05, 0.00012651427823584527, 0.00023838109336793423, 0.0008907323936000466, 0.0014062426052987576, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08771780878305435, 0.00782146118581295, 5.6624849094077945e-05, 1.4731161179781793e-08, 1.0490798558748793e-05, 0.0011058745440095663, 0.00042911627679131925, 5.52300589333754e-05, 3.919872688129544e-05, 2.7502203465701314e-06, 2.5461979475949192e-06, 0.002755212364718318, 0.005722680129110813, 0.0011191823286935687, 0.0015876379329711199, 0.8757358193397522, 0.001080039655789733, 0.0035321444738656282, 0.0015019761631265283, 0.0006949526141397655, 4.334090044721961e-05, 0.00023435342882294208, 6.934279099368723e-06, 2.757802519681718e-08, 9.912479299600818e-07, 0.00019156596681568772, 2.9515393180190586e-05, 5.468214112624992e-06, 2.480577677488327e-06, 1.4659925682281028e-06, 7.256068101924029e-07, 8.567866461817175e-05, 0.004263878334313631, 0.0007048743427731097, 0.0034579597413539886, 0.0, 0.0, 0.0, 0.0, 0.0], [0.018741386011242867, 0.0001317748101428151, 0.00011519038525875658, 8.336405130648927e-07, 3.873432774526009e-07, 4.0343722503166646e-05, 8.927313319873065e-05, 6.575927727681119e-06, 6.646558176726103e-05, 2.0145262169535272e-07, 0.00012498456635512412, 0.00022737613471690565, 0.00035967829171568155, 4.087335037183948e-05, 8.65399488247931e-05, 0.00017062197730410844, 0.9720673561096191, 0.004511047620326281, 0.0017741499468684196, 0.00033421028638258576, 0.0006673677125945687, 8.272402737929951e-06, 8.590817014919594e-05, 4.445200545433181e-07, 1.5395107411109166e-08, 3.916216883226298e-06, 1.8561571778263897e-05, 1.7440333976992406e-06, 2.306471833435353e-05, 6.561277814398636e-07, 5.536517346627079e-05, 1.0027780263044406e-05, 0.00011005577107425779, 2.0988292817492038e-05, 6.0086065786890686e-05, 4.435596929397434e-05, 0.0, 0.0, 0.0, 0.0], [0.02161449007689953, 6.699986261082813e-05, 0.00023383108782581985, 1.9461008093912824e-07, 1.05680453543755e-06, 2.569968273746781e-05, 0.00021046775509603322, 1.7352506347378949e-06, 6.270146968745394e-06, 8.374834692403965e-08, 2.354792513870052e-06, 1.806468026188668e-05, 0.00023376732133328915, 0.0018953320104628801, 0.00020409566059242934, 0.0004746932827401906, 0.0029953878838568926, 0.962553858757019, 0.0069142491556704044, 0.0001976252970052883, 0.00011761276255128905, 1.7810549252317287e-05, 8.193789835786447e-05, 8.319970135062249e-08, 6.341087299688297e-08, 1.5034723901408142e-06, 1.5951758541632444e-05, 1.285200426082156e-07, 7.935356052257703e-07, 1.2686608386047737e-07, 5.14019859565451e-07, 3.882940191601847e-08, 5.885375867364928e-05, 0.0011026912834495306, 9.71672561718151e-05, 8.231383981183171e-05, 0.0007721207803115249, 0.0, 0.0, 0.0], [0.036683376878499985, 5.292138666845858e-06, 1.6796333284219145e-06, 3.225949229967995e-10, 1.7679223418554102e-08, 9.711849997984245e-06, 6.97010455041891e-06, 3.922056293959031e-06, 1.5272446034941822e-05, 6.423179002013057e-07, 3.00263849339899e-07, 2.1527906937990338e-05, 8.620364724265528e-07, 2.37110707530519e-05, 1.133400132857787e-06, 4.3700861169782e-06, 2.1794843632960692e-05, 0.0007244828739203513, 0.956550121307373, 7.606161670992151e-05, 0.005361158866435289, 3.409740747883916e-05, 1.4788669204790494e-06, 1.967607188291254e-09, 6.515270101203896e-09, 7.016132030912559e-07, 1.9082037283624231e-07, 3.6050772678208887e-07, 3.973362709075445e-06, 1.5304426597140264e-06, 1.0371004321996224e-07, 7.635152314833249e-07, 1.8985033989338262e-07, 6.164461410662625e-06, 2.613814103824552e-06, 1.352905542262306e-06, 1.0274186024616938e-05, 0.0004238302353769541, 0.0, 0.0], [0.03937007114291191, 0.00035857962211593986, 7.419281610054895e-05, 4.950996662955731e-05, 4.6530603867722675e-06, 4.697596523328684e-05, 0.00022992271988186985, 1.2504364121923572e-06, 3.72327740478795e-05, 8.911823101698246e-08, 7.931985601317137e-05, 0.000135091133415699, 1.8374767023487948e-05, 0.00010297342669218779, 8.208450162783265e-05, 2.753637636487838e-05, 0.0005808338755741715, 0.0011750266421586275, 0.0009059829753823578, 0.9545682072639465, 7.417569577228278e-05, 0.0005086683668196201, 0.00019448318926151842, 8.633160177851096e-05, 8.95263099209842e-07, 1.4289948921941686e-05, 3.445189940975979e-05, 2.1258644267163618e-07, 1.6950663848547265e-05, 4.816170076082926e-07, 3.18538659485057e-05, 1.3589556147053372e-05, 2.5087487301789224e-05, 5.9819085436174646e-05, 0.0001169012684840709, 1.1341999197611585e-05, 0.0002713195572141558, 0.0002638357982505113, 0.00042732004658319056, 0.0], [0.025401176884770393, 7.441661864504567e-07, 1.0830090104718693e-05, 2.2497226304096785e-09, 9.221913788159952e-10, 8.208573490264826e-06, 2.5448041469644522e-06, 1.0004280426301193e-07, 3.160869891871698e-05, 2.4278301680169534e-07, 5.289186333357065e-07, 1.4262926924857311e-05, 1.5345807469202555e-06, 1.87778505278402e-06, 0.0002028680028161034, 2.1412402020359877e-06, 0.00010114445467479527, 0.0009309852030128241, 0.1016610637307167, 5.551521462621167e-05, 0.8627558350563049, 2.250279521831544e-06, 3.066586941713467e-05, 5.196949981467469e-09, 3.1991624627192294e-11, 8.736405021636529e-08, 1.2241659419487405e-07, 1.2374152813521277e-09, 1.2507486644608434e-05, 4.635763275473437e-07, 1.6297862259762042e-07, 2.719466181133612e-07, 1.1941818911509472e-07, 1.4462575848028791e-07, 5.527890243683942e-05, 3.5639450857161137e-07, 4.933351738145575e-05, 0.0001694788079475984, 0.008456701412796974, 3.871364606311545e-05]]]}\n",
        "    )\n",
        "    </script>"
       ],
       "text/plain": [
-       "<circuitsvis.utils.render.RenderedHTML at 0x7f0f449f7390>"
+       "<circuitsvis.utils.render.RenderedHTML at 0x7f02745a0f10>"
       ]
      },
      "metadata": {},
@@ -1144,17 +1144,17 @@
    "execution_count": 22,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:50.095371Z",
-     "iopub.status.busy": "2024-12-14T01:01:50.094972Z",
-     "iopub.status.idle": "2024-12-14T01:01:53.259149Z",
-     "shell.execute_reply": "2024-12-14T01:01:53.258602Z"
+     "iopub.execute_input": "2024-12-31T02:21:06.480604Z",
+     "iopub.status.busy": "2024-12-31T02:21:06.480219Z",
+     "iopub.status.idle": "2024-12-31T02:21:10.757277Z",
+     "shell.execute_reply": "2024-12-31T02:21:10.756598Z"
     }
    },
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "2b653bed330e47bfaf1eb97604ffb148",
+       "model_id": "16f9e028019c47c19107395141087d9c",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1168,7 +1168,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "2f95bda34e124bdb8ad42b97968c2560",
+       "model_id": "8c41c6eefd75443fa69c9327b4c1ec8e",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1182,7 +1182,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "1bfe4dc8f9d64ed3a70a0e6e41b61d34",
+       "model_id": "6a66981f155d4bf280913b398c75f705",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1196,7 +1196,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "363a53f62bad40f38d9b57415fcdd27a",
+       "model_id": "59a6916e36b44608987b585683320e55",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1210,7 +1210,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "baa802cac1bf4a77bd2d6f2da4e0c206",
+       "model_id": "f462bf0c8aef4d9e9fd54629b8fe4434",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1224,7 +1224,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "85239cb8d6ba4d288b001290e919d816",
+       "model_id": "29b6d4431334417e9690be62ac184ff3",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1253,10 +1253,10 @@
    "execution_count": 23,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:53.261641Z",
-     "iopub.status.busy": "2024-12-14T01:01:53.261263Z",
-     "iopub.status.idle": "2024-12-14T01:01:53.887545Z",
-     "shell.execute_reply": "2024-12-14T01:01:53.886951Z"
+     "iopub.execute_input": "2024-12-31T02:21:10.759709Z",
+     "iopub.status.busy": "2024-12-31T02:21:10.759347Z",
+     "iopub.status.idle": "2024-12-31T02:21:11.367498Z",
+     "shell.execute_reply": "2024-12-31T02:21:11.366898Z"
     }
    },
    "outputs": [
@@ -1267,9 +1267,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"e151a52d-4559-414d-bbd0-4a08a83b57c6\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"e151a52d-4559-414d-bbd0-4a08a83b57c6\")) {                    Plotly.newPlot(                        \"e151a52d-4559-414d-bbd0-4a08a83b57c6\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.010150859132409096,0.00021067557099740952,0.011675543151795864,5.645640158036258e-06,0.0009093668777495623,1.0044651389762294e-05,0.008776976726949215,0.0014797636540606618,0.008503900840878487,0.009458577260375023,0.009176097810268402,0.015341502614319324],[0.0030316568445414305,0.018613651394844055,0.003521384671330452,0.0002105814783135429,0.012771016918122768,0.002262213034555316,0.005199276376515627,0.015398663468658924,0.001596646150574088,7.471267599612474e-05,0.004113694187253714,0.014634193852543831],[0.00835011899471283,0.005591983906924725,0.01345061045140028,0.0016680879052728415,0.019946973770856857,0.004370617680251598,0.009695201180875301,0.001158412080258131,0.01791170984506607,0.01292894221842289,0.021461093798279762,5.112248818717191e-11],[0.005671565420925617,0.20814286172389984,0.8488407135009766,0.01734151318669319,0.017860308289527893,0.014111736789345741,0.020909568294882774,0.1804729551076889,0.01673540472984314,0.0179011020809412,0.9288043975830078,0.4491482973098755],[0.26883465051651,0.23360644280910492,0.08689460158348083,0.011528643779456615,0.07701855152845383,0.024650055915117264,0.6229627132415771,0.025258244946599007,0.06073903664946556,0.6466246247291565,0.017515525221824646,0.054975464940071106],[0.021400248631834984,0.07713880389928818,0.05230269953608513,0.013922988437116146,0.032270368188619614,0.18037253618240356,0.07306526601314545,0.0951840952038765,0.007617602590471506,0.4484187066555023,0.17469297349452972,0.03143536299467087]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Induction Score by Head in Distil GPT-2\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"e0327d47-bf11-4c31-89f2-acefb328a7c7\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"e0327d47-bf11-4c31-89f2-acefb328a7c7\")) {                    Plotly.newPlot(                        \"e0327d47-bf11-4c31-89f2-acefb328a7c7\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.010554764419794083,0.00017109278996940702,0.01157795637845993,2.9588852612505434e-06,0.0007345465710386634,1.1151450053148437e-05,0.00925703626126051,0.001245542080141604,0.00874336902052164,0.009286121465265751,0.009252987802028656,0.01566421426832676],[0.0027158448938280344,0.016322769224643707,0.0025369308423250914,0.0002154479589080438,0.015138639137148857,0.0016251271590590477,0.004624444060027599,0.015645790845155716,0.0013522073859348893,4.790473394677974e-05,0.003898467170074582,0.014824452809989452],[0.009688482619822025,0.007363068405538797,0.011820577085018158,0.0032990179024636745,0.01895608939230442,0.004869616124778986,0.009377731941640377,0.0014279317110776901,0.017006751149892807,0.012095690704882145,0.021088911220431328,8.396961348466903e-09],[0.00907871313393116,0.23671726882457733,0.8545374274253845,0.016443107277154922,0.01815449446439743,0.013023864477872849,0.01957394927740097,0.19624365866184235,0.01637774519622326,0.017492501065135002,0.930077075958252,0.4885671138763428],[0.2790757715702057,0.22406823933124542,0.0913514792919159,0.012907207943499088,0.08906538784503937,0.025342751294374466,0.6347011923789978,0.025462210178375244,0.06470905244350433,0.648553192615509,0.01796729676425457,0.06295257061719894],[0.02277965098619461,0.07785113155841827,0.05185911804437637,0.013325891457498074,0.035574771463871,0.17566177248954773,0.07605704665184021,0.09583288431167603,0.008021574467420578,0.45598646998405457,0.20273584127426147,0.03064807876944542]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Induction Score by Head in Distil GPT-2\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('e151a52d-4559-414d-bbd0-4a08a83b57c6');\n",
+       "var gd = document.getElementById('e0327d47-bf11-4c31-89f2-acefb328a7c7');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -1451,10 +1451,10 @@
    "execution_count": 24,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:53.890023Z",
-     "iopub.status.busy": "2024-12-14T01:01:53.889670Z",
-     "iopub.status.idle": "2024-12-14T01:01:53.893651Z",
-     "shell.execute_reply": "2024-12-14T01:01:53.893084Z"
+     "iopub.execute_input": "2024-12-31T02:21:11.369953Z",
+     "iopub.status.busy": "2024-12-31T02:21:11.369620Z",
+     "iopub.status.idle": "2024-12-31T02:21:11.373683Z",
+     "shell.execute_reply": "2024-12-31T02:21:11.373208Z"
     }
    },
    "outputs": [
@@ -1495,10 +1495,10 @@
    "execution_count": 25,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:53.895827Z",
-     "iopub.status.busy": "2024-12-14T01:01:53.895486Z",
-     "iopub.status.idle": "2024-12-14T01:01:53.899011Z",
-     "shell.execute_reply": "2024-12-14T01:01:53.898465Z"
+     "iopub.execute_input": "2024-12-31T02:21:11.375678Z",
+     "iopub.status.busy": "2024-12-31T02:21:11.375277Z",
+     "iopub.status.idle": "2024-12-31T02:21:11.379030Z",
+     "shell.execute_reply": "2024-12-31T02:21:11.378468Z"
     }
    },
    "outputs": [
@@ -1540,10 +1540,10 @@
    "execution_count": 26,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:53.901258Z",
-     "iopub.status.busy": "2024-12-14T01:01:53.900920Z",
-     "iopub.status.idle": "2024-12-14T01:01:53.970759Z",
-     "shell.execute_reply": "2024-12-14T01:01:53.970165Z"
+     "iopub.execute_input": "2024-12-31T02:21:11.381085Z",
+     "iopub.status.busy": "2024-12-31T02:21:11.380739Z",
+     "iopub.status.idle": "2024-12-31T02:21:11.447606Z",
+     "shell.execute_reply": "2024-12-31T02:21:11.447031Z"
     }
    },
    "outputs": [
@@ -1637,10 +1637,10 @@
    "execution_count": 27,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:53.973146Z",
-     "iopub.status.busy": "2024-12-14T01:01:53.972816Z",
-     "iopub.status.idle": "2024-12-14T01:01:53.980352Z",
-     "shell.execute_reply": "2024-12-14T01:01:53.979763Z"
+     "iopub.execute_input": "2024-12-31T02:21:11.450000Z",
+     "iopub.status.busy": "2024-12-31T02:21:11.449666Z",
+     "iopub.status.idle": "2024-12-31T02:21:11.457135Z",
+     "shell.execute_reply": "2024-12-31T02:21:11.456551Z"
     }
    },
    "outputs": [],
@@ -1654,10 +1654,10 @@
    "execution_count": 28,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:53.982600Z",
-     "iopub.status.busy": "2024-12-14T01:01:53.982257Z",
-     "iopub.status.idle": "2024-12-14T01:01:53.987655Z",
-     "shell.execute_reply": "2024-12-14T01:01:53.987079Z"
+     "iopub.execute_input": "2024-12-31T02:21:11.459407Z",
+     "iopub.status.busy": "2024-12-31T02:21:11.459011Z",
+     "iopub.status.idle": "2024-12-31T02:21:11.465771Z",
+     "shell.execute_reply": "2024-12-31T02:21:11.465212Z"
     }
    },
    "outputs": [
@@ -1735,10 +1735,10 @@
    "execution_count": 29,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:53.989809Z",
-     "iopub.status.busy": "2024-12-14T01:01:53.989465Z",
-     "iopub.status.idle": "2024-12-14T01:01:53.993884Z",
-     "shell.execute_reply": "2024-12-14T01:01:53.993446Z"
+     "iopub.execute_input": "2024-12-31T02:21:11.467753Z",
+     "iopub.status.busy": "2024-12-31T02:21:11.467444Z",
+     "iopub.status.idle": "2024-12-31T02:21:11.471612Z",
+     "shell.execute_reply": "2024-12-31T02:21:11.471176Z"
     }
    },
    "outputs": [
@@ -1803,10 +1803,10 @@
    "execution_count": 30,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:53.996070Z",
-     "iopub.status.busy": "2024-12-14T01:01:53.995639Z",
-     "iopub.status.idle": "2024-12-14T01:01:53.999873Z",
-     "shell.execute_reply": "2024-12-14T01:01:53.999303Z"
+     "iopub.execute_input": "2024-12-31T02:21:11.473791Z",
+     "iopub.status.busy": "2024-12-31T02:21:11.473360Z",
+     "iopub.status.idle": "2024-12-31T02:21:11.477507Z",
+     "shell.execute_reply": "2024-12-31T02:21:11.476948Z"
     }
    },
    "outputs": [
@@ -1836,10 +1836,10 @@
    "execution_count": 31,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:54.002099Z",
-     "iopub.status.busy": "2024-12-14T01:01:54.001703Z",
-     "iopub.status.idle": "2024-12-14T01:01:54.005196Z",
-     "shell.execute_reply": "2024-12-14T01:01:54.004759Z"
+     "iopub.execute_input": "2024-12-31T02:21:11.479502Z",
+     "iopub.status.busy": "2024-12-31T02:21:11.479196Z",
+     "iopub.status.idle": "2024-12-31T02:21:11.482765Z",
+     "shell.execute_reply": "2024-12-31T02:21:11.482232Z"
     }
    },
    "outputs": [
@@ -1876,10 +1876,10 @@
    "execution_count": 32,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:54.007056Z",
-     "iopub.status.busy": "2024-12-14T01:01:54.006893Z",
-     "iopub.status.idle": "2024-12-14T01:01:54.010707Z",
-     "shell.execute_reply": "2024-12-14T01:01:54.010246Z"
+     "iopub.execute_input": "2024-12-31T02:21:11.484855Z",
+     "iopub.status.busy": "2024-12-31T02:21:11.484546Z",
+     "iopub.status.idle": "2024-12-31T02:21:11.488058Z",
+     "shell.execute_reply": "2024-12-31T02:21:11.487619Z"
     }
    },
    "outputs": [
@@ -1919,10 +1919,10 @@
    "execution_count": 33,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:54.013081Z",
-     "iopub.status.busy": "2024-12-14T01:01:54.012542Z",
-     "iopub.status.idle": "2024-12-14T01:01:54.100258Z",
-     "shell.execute_reply": "2024-12-14T01:01:54.099682Z"
+     "iopub.execute_input": "2024-12-31T02:21:11.490041Z",
+     "iopub.status.busy": "2024-12-31T02:21:11.489714Z",
+     "iopub.status.idle": "2024-12-31T02:21:11.572950Z",
+     "shell.execute_reply": "2024-12-31T02:21:11.572365Z"
     }
    },
    "outputs": [
@@ -1930,13 +1930,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Probability tensor shape [batch, position, d_vocab] == torch.Size([1, 8, 50257])\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
+      "Probability tensor shape [batch, position, d_vocab] == torch.Size([1, 8, 50257])\n",
       "| The| probability: 11.98%\n"
      ]
     }
@@ -1965,10 +1959,10 @@
    "execution_count": 34,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:54.102428Z",
-     "iopub.status.busy": "2024-12-14T01:01:54.102246Z",
-     "iopub.status.idle": "2024-12-14T01:01:54.105837Z",
-     "shell.execute_reply": "2024-12-14T01:01:54.105312Z"
+     "iopub.execute_input": "2024-12-31T02:21:11.575202Z",
+     "iopub.status.busy": "2024-12-31T02:21:11.575030Z",
+     "iopub.status.idle": "2024-12-31T02:21:11.578555Z",
+     "shell.execute_reply": "2024-12-31T02:21:11.578105Z"
     }
    },
    "outputs": [
@@ -2003,10 +1997,10 @@
    "execution_count": 35,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:54.107939Z",
-     "iopub.status.busy": "2024-12-14T01:01:54.107614Z",
-     "iopub.status.idle": "2024-12-14T01:01:54.111996Z",
-     "shell.execute_reply": "2024-12-14T01:01:54.111534Z"
+     "iopub.execute_input": "2024-12-31T02:21:11.580549Z",
+     "iopub.status.busy": "2024-12-31T02:21:11.580229Z",
+     "iopub.status.idle": "2024-12-31T02:21:11.584406Z",
+     "shell.execute_reply": "2024-12-31T02:21:11.583987Z"
     }
    },
    "outputs": [
@@ -2036,10 +2030,10 @@
    "execution_count": 36,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:54.114136Z",
-     "iopub.status.busy": "2024-12-14T01:01:54.113818Z",
-     "iopub.status.idle": "2024-12-14T01:01:54.118199Z",
-     "shell.execute_reply": "2024-12-14T01:01:54.117648Z"
+     "iopub.execute_input": "2024-12-31T02:21:11.586445Z",
+     "iopub.status.busy": "2024-12-31T02:21:11.586126Z",
+     "iopub.status.idle": "2024-12-31T02:21:11.590167Z",
+     "shell.execute_reply": "2024-12-31T02:21:11.589731Z"
     }
    },
    "outputs": [
@@ -2075,10 +2069,10 @@
    "execution_count": 37,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:54.120410Z",
-     "iopub.status.busy": "2024-12-14T01:01:54.120080Z",
-     "iopub.status.idle": "2024-12-14T01:01:54.124019Z",
-     "shell.execute_reply": "2024-12-14T01:01:54.123552Z"
+     "iopub.execute_input": "2024-12-31T02:21:11.592212Z",
+     "iopub.status.busy": "2024-12-31T02:21:11.591791Z",
+     "iopub.status.idle": "2024-12-31T02:21:11.595687Z",
+     "shell.execute_reply": "2024-12-31T02:21:11.595261Z"
     }
    },
    "outputs": [
@@ -2126,10 +2120,10 @@
    "execution_count": 38,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:54.126145Z",
-     "iopub.status.busy": "2024-12-14T01:01:54.125812Z",
-     "iopub.status.idle": "2024-12-14T01:01:54.343747Z",
-     "shell.execute_reply": "2024-12-14T01:01:54.343133Z"
+     "iopub.execute_input": "2024-12-31T02:21:11.597571Z",
+     "iopub.status.busy": "2024-12-31T02:21:11.597408Z",
+     "iopub.status.idle": "2024-12-31T02:21:11.803117Z",
+     "shell.execute_reply": "2024-12-31T02:21:11.802524Z"
     }
    },
    "outputs": [
@@ -2179,10 +2173,10 @@
    "execution_count": 39,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:54.346431Z",
-     "iopub.status.busy": "2024-12-14T01:01:54.346064Z",
-     "iopub.status.idle": "2024-12-14T01:01:54.545626Z",
-     "shell.execute_reply": "2024-12-14T01:01:54.545021Z"
+     "iopub.execute_input": "2024-12-31T02:21:11.805474Z",
+     "iopub.status.busy": "2024-12-31T02:21:11.805123Z",
+     "iopub.status.idle": "2024-12-31T02:21:11.998355Z",
+     "shell.execute_reply": "2024-12-31T02:21:11.997764Z"
     }
    },
    "outputs": [
@@ -2197,14 +2191,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Logit difference without BOS: 2.782"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\n"
+      "Logit difference without BOS: 2.782\n"
      ]
     }
    ],
@@ -2232,10 +2219,10 @@
    "execution_count": 40,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:54.548090Z",
-     "iopub.status.busy": "2024-12-14T01:01:54.547737Z",
-     "iopub.status.idle": "2024-12-14T01:01:54.551966Z",
-     "shell.execute_reply": "2024-12-14T01:01:54.551372Z"
+     "iopub.execute_input": "2024-12-31T02:21:12.000814Z",
+     "iopub.status.busy": "2024-12-31T02:21:12.000468Z",
+     "iopub.status.idle": "2024-12-31T02:21:12.004466Z",
+     "shell.execute_reply": "2024-12-31T02:21:12.003902Z"
     }
    },
    "outputs": [
@@ -2292,10 +2279,10 @@
    "execution_count": 41,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:54.554257Z",
-     "iopub.status.busy": "2024-12-14T01:01:54.553915Z",
-     "iopub.status.idle": "2024-12-14T01:01:54.559609Z",
-     "shell.execute_reply": "2024-12-14T01:01:54.559136Z"
+     "iopub.execute_input": "2024-12-31T02:21:12.006510Z",
+     "iopub.status.busy": "2024-12-31T02:21:12.006206Z",
+     "iopub.status.idle": "2024-12-31T02:21:12.011745Z",
+     "shell.execute_reply": "2024-12-31T02:21:12.011254Z"
     }
    },
    "outputs": [
@@ -2337,10 +2324,10 @@
    "execution_count": 42,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:54.561788Z",
-     "iopub.status.busy": "2024-12-14T01:01:54.561461Z",
-     "iopub.status.idle": "2024-12-14T01:01:54.566401Z",
-     "shell.execute_reply": "2024-12-14T01:01:54.565937Z"
+     "iopub.execute_input": "2024-12-31T02:21:12.013860Z",
+     "iopub.status.busy": "2024-12-31T02:21:12.013698Z",
+     "iopub.status.idle": "2024-12-31T02:21:12.019174Z",
+     "shell.execute_reply": "2024-12-31T02:21:12.018583Z"
     }
    },
    "outputs": [
@@ -2382,10 +2369,10 @@
    "execution_count": 43,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:54.568434Z",
-     "iopub.status.busy": "2024-12-14T01:01:54.568104Z",
-     "iopub.status.idle": "2024-12-14T01:01:54.573557Z",
-     "shell.execute_reply": "2024-12-14T01:01:54.573073Z"
+     "iopub.execute_input": "2024-12-31T02:21:12.021351Z",
+     "iopub.status.busy": "2024-12-31T02:21:12.020961Z",
+     "iopub.status.idle": "2024-12-31T02:21:12.027750Z",
+     "shell.execute_reply": "2024-12-31T02:21:12.027228Z"
     }
    },
    "outputs": [
@@ -2424,10 +2411,10 @@
    "execution_count": 44,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:54.575636Z",
-     "iopub.status.busy": "2024-12-14T01:01:54.575308Z",
-     "iopub.status.idle": "2024-12-14T01:01:54.578709Z",
-     "shell.execute_reply": "2024-12-14T01:01:54.578240Z"
+     "iopub.execute_input": "2024-12-31T02:21:12.030023Z",
+     "iopub.status.busy": "2024-12-31T02:21:12.029712Z",
+     "iopub.status.idle": "2024-12-31T02:21:12.034564Z",
+     "shell.execute_reply": "2024-12-31T02:21:12.034096Z"
     }
    },
    "outputs": [
@@ -2474,10 +2461,10 @@
    "execution_count": 45,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:54.580629Z",
-     "iopub.status.busy": "2024-12-14T01:01:54.580461Z",
-     "iopub.status.idle": "2024-12-14T01:01:54.587025Z",
-     "shell.execute_reply": "2024-12-14T01:01:54.586507Z"
+     "iopub.execute_input": "2024-12-31T02:21:12.036656Z",
+     "iopub.status.busy": "2024-12-31T02:21:12.036488Z",
+     "iopub.status.idle": "2024-12-31T02:21:12.043644Z",
+     "shell.execute_reply": "2024-12-31T02:21:12.043097Z"
     }
    },
    "outputs": [
@@ -2499,10 +2486,10 @@
    "execution_count": 46,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:54.589336Z",
-     "iopub.status.busy": "2024-12-14T01:01:54.588929Z",
-     "iopub.status.idle": "2024-12-14T01:01:54.698460Z",
-     "shell.execute_reply": "2024-12-14T01:01:54.697829Z"
+     "iopub.execute_input": "2024-12-31T02:21:12.045888Z",
+     "iopub.status.busy": "2024-12-31T02:21:12.045522Z",
+     "iopub.status.idle": "2024-12-31T02:21:12.166004Z",
+     "shell.execute_reply": "2024-12-31T02:21:12.165455Z"
     }
    },
    "outputs": [
@@ -2526,10 +2513,10 @@
    "execution_count": 47,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:54.700835Z",
-     "iopub.status.busy": "2024-12-14T01:01:54.700449Z",
-     "iopub.status.idle": "2024-12-14T01:01:54.731325Z",
-     "shell.execute_reply": "2024-12-14T01:01:54.730826Z"
+     "iopub.execute_input": "2024-12-31T02:21:12.168050Z",
+     "iopub.status.busy": "2024-12-31T02:21:12.167864Z",
+     "iopub.status.idle": "2024-12-31T02:21:12.197547Z",
+     "shell.execute_reply": "2024-12-31T02:21:12.196930Z"
     }
    },
    "outputs": [
@@ -2540,9 +2527,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"458e8707-ccf3-4567-a8b0-f931c572b246\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"458e8707-ccf3-4567-a8b0-f931c572b246\")) {                    Plotly.newPlot(                        \"458e8707-ccf3-4567-a8b0-f931c572b246\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.7775008678436279,0.35272663831710815,0.25961872935295105,0.6670258045196533,0.838425874710083,0.5584433078765869,0.844474732875824,0.41379114985466003,0.244889497756958,0.02815753035247326,0.3584096431732178,0.16288283467292786],[-0.4541913866996765,-0.6529324650764465,-0.5484569668769836,-0.7990368604660034,-0.7736426591873169,-0.8522582650184631,0.977432370185852,0.6626251935958862,-0.7303223609924316,-0.7007019519805908,-0.6946623921394348,-0.9996723532676697],[-0.7837162613868713,0.8967758417129517,0.4750956892967224,-0.6671973466873169,0.7881463170051575,-0.8547748923301697,-0.9054183959960938,-0.5749384760856628,-0.321751207113266,-0.0285941194742918,-0.9247617125511169,-0.9699268937110901],[0.5864036083221436,-0.76143479347229,0.5971695184707642,0.7854391932487488,-0.8788884878158569,0.3908745348453522,0.0447387658059597,0.1102800965309143,-0.8169987201690674,0.2212953418493271,-0.9939578771591187,0.5774401426315308],[0.5254791378974915,0.30490121245384216,-0.10729170590639114,0.9433152675628662,-0.9314427971839905,0.5273631811141968,-0.4264712929725647,-0.9984429478645325,0.5296757817268372,0.8604294657707214,-0.8895052075386047,0.9556970000267029],[0.6629188656806946,0.4295697808265686,0.9736858010292053,0.655548095703125,0.12201863527297974,0.7442769408226013,0.5037954449653625,0.95253586769104,-0.6507166624069214,-0.9316278100013733,0.9791510701179504,-0.9972586035728455],[0.9613031148910522,0.7501779794692993,-0.3806658983230591,0.6429785490036011,0.9557769298553467,-0.9428837895393372,-0.9948079586029053,0.7852989435195923,0.9657301306724548,0.7073014974594116,0.36872273683547974,0.8128010034561157],[0.9659482836723328,0.9730120301246643,0.31900620460510254,-0.30290529131889343,0.9790952801704407,0.9357922673225403,-0.5550314784049988,-0.0054661668837070465,0.986777663230896,0.8249568343162537,0.5664296746253967,0.1000528484582901],[-0.9464486837387085,-0.25471991300582886,0.6522327065467834,0.14152583479881287,0.9884141683578491,0.9860584735870361,0.6949271559715271,0.9901811480522156,0.9791203141212463,-0.23595543205738068,-0.982071042060852,0.6506688594818115],[0.9895945191383362,-0.291781485080719,0.9714024662971497,0.9951602220535278,0.18783727288246155,-0.9460937976837158,0.4780191481113434,-0.2489192634820938,0.9437099099159241,0.11866225302219391,0.9941242933273315,-0.38088199496269226],[0.9564487338066101,0.5542722344398499,0.42118069529533386,0.6628788113594055,0.8659593462944031,0.9937117695808411,0.9069075584411621,0.3981107473373413,-0.4134218096733093,0.9971914887428284,0.3459664583206177,0.9938657283782959],[0.5891268253326416,0.9313738942146301,0.9268401861190796,0.9993563890457153,0.6227542161941528,0.8463947772979736,0.6584343910217285,0.8423123955726624,0.2978499233722687,0.8728678822517395,0.9963143467903137,0.9867526292800903]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0,\"cmin\":-1.0,\"cmax\":1.0},\"title\":{\"text\":\"OV Copying Score for each head in GPT-2 Small\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"d3f848b1-d2b5-480a-aea1-fbcc08758557\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"d3f848b1-d2b5-480a-aea1-fbcc08758557\")) {                    Plotly.newPlot(                        \"d3f848b1-d2b5-480a-aea1-fbcc08758557\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.7775008678436279,0.35272663831710815,0.25961872935295105,0.6670258045196533,0.838425874710083,0.5584433078765869,0.844474732875824,0.41379114985466003,0.244889497756958,0.02815753035247326,0.3584096431732178,0.16288283467292786],[-0.4541913866996765,-0.6529324650764465,-0.5484569668769836,-0.7990368604660034,-0.7736426591873169,-0.8522582650184631,0.977432370185852,0.6626251935958862,-0.7303223609924316,-0.7007019519805908,-0.6946623921394348,-0.9996723532676697],[-0.7837162613868713,0.8967758417129517,0.4750956892967224,-0.6671973466873169,0.7881463170051575,-0.8547748923301697,-0.9054183959960938,-0.5749384760856628,-0.321751207113266,-0.0285941194742918,-0.9247617125511169,-0.9699268937110901],[0.5864036083221436,-0.76143479347229,0.5971695184707642,0.7854391932487488,-0.8788884878158569,0.3908745348453522,0.0447387658059597,0.1102800965309143,-0.8169987201690674,0.2212953418493271,-0.9939578771591187,0.5774401426315308],[0.5254791378974915,0.30490121245384216,-0.10729170590639114,0.9433152675628662,-0.9314427971839905,0.5273631811141968,-0.4264712929725647,-0.9984429478645325,0.5296757817268372,0.8604294657707214,-0.8895052075386047,0.9556970000267029],[0.6629188656806946,0.4295697808265686,0.9736858010292053,0.655548095703125,0.12201863527297974,0.7442769408226013,0.5037954449653625,0.95253586769104,-0.6507166624069214,-0.9316278100013733,0.9791510701179504,-0.9972586035728455],[0.9613031148910522,0.7501779794692993,-0.3806658983230591,0.6429785490036011,0.9557769298553467,-0.9428837895393372,-0.9948079586029053,0.7852989435195923,0.9657301306724548,0.7073014974594116,0.36872273683547974,0.8128010034561157],[0.9659482836723328,0.9730120301246643,0.31900620460510254,-0.30290529131889343,0.9790952801704407,0.9357922673225403,-0.5550314784049988,-0.0054661668837070465,0.986777663230896,0.8249568343162537,0.5664296746253967,0.1000528484582901],[-0.9464486837387085,-0.25471991300582886,0.6522327065467834,0.14152583479881287,0.9884141683578491,0.9860584735870361,0.6949271559715271,0.9901811480522156,0.9791203141212463,-0.23595543205738068,-0.982071042060852,0.6506688594818115],[0.9895945191383362,-0.291781485080719,0.9714024662971497,0.9951602220535278,0.18783727288246155,-0.9460937976837158,0.4780191481113434,-0.2489192634820938,0.9437099099159241,0.11866225302219391,0.9941242933273315,-0.38088199496269226],[0.9564487338066101,0.5542722344398499,0.42118069529533386,0.6628788113594055,0.8659593462944031,0.9937117695808411,0.9069075584411621,0.3981107473373413,-0.4134218096733093,0.9971914887428284,0.3459664583206177,0.9938657283782959],[0.5891268253326416,0.9313738942146301,0.9268401861190796,0.9993563890457153,0.6227542161941528,0.8463947772979736,0.6584343910217285,0.8423123955726624,0.2978499233722687,0.8728678822517395,0.9963143467903137,0.9867526292800903]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0,\"cmin\":-1.0,\"cmax\":1.0},\"title\":{\"text\":\"OV Copying Score for each head in GPT-2 Small\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('458e8707-ccf3-4567-a8b0-f931c572b246');\n",
+       "var gd = document.getElementById('d3f848b1-d2b5-480a-aea1-fbcc08758557');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -2590,10 +2577,10 @@
    "execution_count": 48,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:54.733451Z",
-     "iopub.status.busy": "2024-12-14T01:01:54.733127Z",
-     "iopub.status.idle": "2024-12-14T01:01:54.763618Z",
-     "shell.execute_reply": "2024-12-14T01:01:54.762996Z"
+     "iopub.execute_input": "2024-12-31T02:21:12.199438Z",
+     "iopub.status.busy": "2024-12-31T02:21:12.199271Z",
+     "iopub.status.idle": "2024-12-31T02:21:12.228189Z",
+     "shell.execute_reply": "2024-12-31T02:21:12.227721Z"
     }
    },
    "outputs": [
@@ -2604,9 +2591,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"79eef653-2ecc-4731-aedd-e4b748085838\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"79eef653-2ecc-4731-aedd-e4b748085838\")) {                    Plotly.newPlot(                        \"79eef653-2ecc-4731-aedd-e4b748085838\",                        [{\"hovertemplate\":\"Real=%{x}\\u003cbr\\u003eImaginary=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"\",\"marker\":{\"color\":\"#636efa\",\"symbol\":\"circle\"},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[-2.139726161956787,1.4152659177780151,3.4444546699523926,4.0276689529418945,8.882657051086426,4.866776943206787,4.866776943206787,4.843708038330078,4.843708038330078,8.477534294128418,8.21679973602295,8.21679973602295,5.078621864318848,7.855459213256836,7.855459213256836,5.365771770477295,5.365771770477295,5.563427448272705,5.563427448272705,5.421727657318115,7.769131660461426,7.769131660461426,7.042290210723877,7.042290210723877,5.675149917602539,5.675149917602539,7.6785993576049805,7.6785993576049805,6.573331356048584,6.573331356048584,7.67292594909668,7.172202110290527,7.172202110290527,7.423615455627441,7.423615455627441,7.4708147048950195,6.08908748626709,6.08908748626709,6.306832313537598,6.306832313537598,6.511747360229492,6.511747360229492,5.955250263214111,5.955250263214111,5.858802795410156,5.858802795410156,7.147878170013428,7.147878170013428,7.185699939727783,7.185699939727783,6.6706037521362305,6.6706037521362305,6.7359819412231445,6.7359819412231445,6.149754524230957,6.149754524230957,6.288780212402344,6.288780212402344,6.344789028167725,6.625576972961426,6.625576972961426,6.899187088012695,6.899187088012695,6.856410026550293],\"xaxis\":\"x\",\"y\":[0.0,0.0,0.0,0.0,0.0,0.41851967573165894,-0.41851967573165894,0.09079498052597046,-0.09079498052597046,0.0,0.40868431329727173,-0.40868431329727173,0.0,0.7007191777229309,-0.7007191777229309,0.46421778202056885,-0.46421778202056885,0.5558270215988159,-0.5558270215988159,0.0,0.47056713700294495,-0.47056713700294495,1.029872179031372,-1.029872179031372,0.48252731561660767,-0.48252731561660767,0.33564886450767517,-0.33564886450767517,0.9988701343536377,-0.9988701343536377,0.0,0.7531797885894775,-0.7531797885894775,0.425758957862854,-0.425758957862854,0.0,0.6436269283294678,-0.6436269283294678,0.7701666951179504,-0.7701666951179504,0.7558029294013977,-0.7558029294013977,0.2591177523136139,-0.2591177523136139,0.013043581508100033,-0.013043581508100033,0.40166139602661133,-0.40166139602661133,0.28192487359046936,-0.28192487359046936,0.6146230697631836,-0.6146230697631836,0.5391324758529663,-0.5391324758529663,0.2823374569416046,-0.2823374569416046,0.35283783078193665,-0.35283783078193665,0.0,0.24868342280387878,-0.24868342280387878,0.15545885264873505,-0.15545885264873505,0.0],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Real\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Imaginary\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Eigenvalues of Head L11H11 of GPT-2 Small\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"e046ce15-f732-4def-bd7e-56de079b2f60\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"e046ce15-f732-4def-bd7e-56de079b2f60\")) {                    Plotly.newPlot(                        \"e046ce15-f732-4def-bd7e-56de079b2f60\",                        [{\"hovertemplate\":\"Real=%{x}\\u003cbr\\u003eImaginary=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"\",\"marker\":{\"color\":\"#636efa\",\"symbol\":\"circle\"},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[-2.139726161956787,1.4152659177780151,3.4444546699523926,4.0276689529418945,8.882657051086426,4.866776943206787,4.866776943206787,4.843708038330078,4.843708038330078,8.477534294128418,8.21679973602295,8.21679973602295,5.078621864318848,7.855459213256836,7.855459213256836,5.365771770477295,5.365771770477295,5.563427448272705,5.563427448272705,5.421727657318115,7.769131660461426,7.769131660461426,7.042290210723877,7.042290210723877,5.675149917602539,5.675149917602539,7.6785993576049805,7.6785993576049805,6.573331356048584,6.573331356048584,7.67292594909668,7.172202110290527,7.172202110290527,7.423615455627441,7.423615455627441,7.4708147048950195,6.08908748626709,6.08908748626709,6.306832313537598,6.306832313537598,6.511747360229492,6.511747360229492,5.955250263214111,5.955250263214111,5.858802795410156,5.858802795410156,7.147878170013428,7.147878170013428,7.185699939727783,7.185699939727783,6.6706037521362305,6.6706037521362305,6.7359819412231445,6.7359819412231445,6.149754524230957,6.149754524230957,6.288780212402344,6.288780212402344,6.344789028167725,6.625576972961426,6.625576972961426,6.899187088012695,6.899187088012695,6.856410026550293],\"xaxis\":\"x\",\"y\":[0.0,0.0,0.0,0.0,0.0,0.41851967573165894,-0.41851967573165894,0.09079498052597046,-0.09079498052597046,0.0,0.40868431329727173,-0.40868431329727173,0.0,0.7007191777229309,-0.7007191777229309,0.46421778202056885,-0.46421778202056885,0.5558270215988159,-0.5558270215988159,0.0,0.47056713700294495,-0.47056713700294495,1.029872179031372,-1.029872179031372,0.48252731561660767,-0.48252731561660767,0.33564886450767517,-0.33564886450767517,0.9988701343536377,-0.9988701343536377,0.0,0.7531797885894775,-0.7531797885894775,0.425758957862854,-0.425758957862854,0.0,0.6436269283294678,-0.6436269283294678,0.7701666951179504,-0.7701666951179504,0.7558029294013977,-0.7558029294013977,0.2591177523136139,-0.2591177523136139,0.013043581508100033,-0.013043581508100033,0.40166139602661133,-0.40166139602661133,0.28192487359046936,-0.28192487359046936,0.6146230697631836,-0.6146230697631836,0.5391324758529663,-0.5391324758529663,0.2823374569416046,-0.2823374569416046,0.35283783078193665,-0.35283783078193665,0.0,0.24868342280387878,-0.24868342280387878,0.15545885264873505,-0.15545885264873505,0.0],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Real\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Imaginary\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Eigenvalues of Head L11H11 of GPT-2 Small\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('79eef653-2ecc-4731-aedd-e4b748085838');\n",
+       "var gd = document.getElementById('e046ce15-f732-4def-bd7e-56de079b2f60');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -2653,10 +2640,10 @@
    "execution_count": 49,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:01:54.765962Z",
-     "iopub.status.busy": "2024-12-14T01:01:54.765604Z",
-     "iopub.status.idle": "2024-12-14T01:02:05.135799Z",
-     "shell.execute_reply": "2024-12-14T01:02:05.135140Z"
+     "iopub.execute_input": "2024-12-31T02:21:12.230091Z",
+     "iopub.status.busy": "2024-12-31T02:21:12.229918Z",
+     "iopub.status.idle": "2024-12-31T02:21:22.297759Z",
+     "shell.execute_reply": "2024-12-31T02:21:22.297167Z"
     }
    },
    "outputs": [
@@ -2678,10 +2665,10 @@
    "execution_count": 50,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:02:05.138206Z",
-     "iopub.status.busy": "2024-12-14T01:02:05.137845Z",
-     "iopub.status.idle": "2024-12-14T01:02:05.685306Z",
-     "shell.execute_reply": "2024-12-14T01:02:05.684702Z"
+     "iopub.execute_input": "2024-12-31T02:21:22.299933Z",
+     "iopub.status.busy": "2024-12-31T02:21:22.299756Z",
+     "iopub.status.idle": "2024-12-31T02:21:22.821584Z",
+     "shell.execute_reply": "2024-12-31T02:21:22.820971Z"
     }
    },
    "outputs": [
@@ -2705,10 +2692,10 @@
    "execution_count": 51,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:02:05.687795Z",
-     "iopub.status.busy": "2024-12-14T01:02:05.687450Z",
-     "iopub.status.idle": "2024-12-14T01:02:05.718903Z",
-     "shell.execute_reply": "2024-12-14T01:02:05.718334Z"
+     "iopub.execute_input": "2024-12-31T02:21:22.823699Z",
+     "iopub.status.busy": "2024-12-31T02:21:22.823527Z",
+     "iopub.status.idle": "2024-12-31T02:21:22.853024Z",
+     "shell.execute_reply": "2024-12-31T02:21:22.852564Z"
     }
    },
    "outputs": [
@@ -2719,9 +2706,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"b62ac579-268a-4cb8-a4db-de9c144e5855\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"b62ac579-268a-4cb8-a4db-de9c144e5855\")) {                    Plotly.newPlot(                        \"b62ac579-268a-4cb8-a4db-de9c144e5855\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.8356367349624634,0.5853534936904907,0.5105841159820557,0.7843377590179443,0.8644161224365234,0.7026589512825012,0.8969924449920654,0.5868823528289795,0.42486509680747986,-0.16337518393993378,0.46268585324287415,0.276053786277771],[-0.052920110523700714,-0.31773144006729126,-0.4810579717159271,-0.7838066220283508,-0.6360211968421936,-0.77586829662323,0.9681803584098816,0.8119116425514221,-0.7510464787483215,-0.6878445744514465,-0.6429887413978577,-0.9985856413841248],[-0.6598325967788696,0.9152501821517944,0.5461500883102417,-0.48743969202041626,0.7720563411712646,-0.7541061639785767,-0.8472450971603394,-0.6948987245559692,-0.1557510495185852,0.2444225549697876,-0.9106622338294983,-0.9439151287078857],[0.648689866065979,-0.5592910647392273,0.5935593247413635,0.7843040823936462,-0.8150346279144287,0.613004744052887,0.16785882413387299,0.35195910930633545,-0.6837262511253357,0.2223764955997467,-0.9929219484329224,0.6535818576812744],[0.5740953087806702,0.3640134036540985,0.09609051048755646,0.9359624981880188,-0.9228776097297668,0.6191076636314392,-0.33572638034820557,-0.998464822769165,0.6448632478713989,0.8468661308288574,-0.7557656764984131,0.9527971744537354],[0.7326544523239136,0.5324169397354126,0.9732670187950134,0.7239246964454651,0.2553895115852356,0.8158416152000427,0.665579080581665,0.9287100434303284,-0.5660436153411865,-0.8908745050430298,0.9834234118461609,-0.9981179237365723],[0.9698692560195923,0.7439671754837036,-0.35639333724975586,0.6022987365722656,0.9708116054534912,-0.9278275966644287,-0.9962316155433655,0.834520697593689,0.9714328050613403,0.8158544898033142,0.5902575850486755,0.8199343681335449],[0.9820227026939392,0.9859329462051392,0.5152460932731628,-0.5610517263412476,0.9663666486740112,0.9495157599449158,-0.5204814076423645,0.3104752004146576,0.9859083890914917,0.7797460556030273,0.6738530397415161,0.39197394251823425],[-0.906204104423523,0.11750960350036621,0.8077874779701233,0.416930615901947,0.9829014539718628,0.9902300834655762,0.7847104668617249,0.9945629835128784,0.9868024587631226,-0.26804426312446594,-0.9908867478370667,0.745792806148529],[0.9906190633773804,-0.18231146037578583,0.9757838249206543,0.9986750483512878,0.2544330954551697,-0.954406201839447,0.586924135684967,-0.23537978529930115,0.9550502896308899,0.2551196813583374,0.9929869771003723,0.0905260294675827],[0.9707273244857788,0.6956090927124023,0.6280022263526917,0.7902868390083313,0.9343841671943665,0.9895793795585632,0.9436282515525818,-0.10834990441799164,-0.3431110680103302,0.9986709952354431,0.508673906326294,0.9949509501457214],[0.8283132910728455,0.9432437419891357,0.9491764903068542,0.9995353817939758,0.5712320804595947,0.8055236339569092,0.6781865954399109,0.8272573351860046,0.8314797282218933,0.8778655529022217,0.9944959282875061,0.997386634349823]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0,\"cmin\":-1.0,\"cmax\":1.0},\"title\":{\"text\":\"OV Copying Score for each head in GPT-2 Small\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"bac66523-6094-49ba-be5d-8fd8a0f66934\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"bac66523-6094-49ba-be5d-8fd8a0f66934\")) {                    Plotly.newPlot(                        \"bac66523-6094-49ba-be5d-8fd8a0f66934\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.8356367349624634,0.5853534936904907,0.5105841159820557,0.7843377590179443,0.8644161224365234,0.7026589512825012,0.8969924449920654,0.5868823528289795,0.42486509680747986,-0.16337518393993378,0.46268585324287415,0.276053786277771],[-0.052920110523700714,-0.31773144006729126,-0.4810579717159271,-0.7838066220283508,-0.6360211968421936,-0.77586829662323,0.9681803584098816,0.8119116425514221,-0.7510464787483215,-0.6878445744514465,-0.6429887413978577,-0.9985856413841248],[-0.6598325967788696,0.9152501821517944,0.5461500883102417,-0.48743969202041626,0.7720563411712646,-0.7541061639785767,-0.8472450971603394,-0.6948987245559692,-0.1557510495185852,0.2444225549697876,-0.9106622338294983,-0.9439151287078857],[0.648689866065979,-0.5592910647392273,0.5935593247413635,0.7843040823936462,-0.8150346279144287,0.613004744052887,0.16785882413387299,0.35195910930633545,-0.6837262511253357,0.2223764955997467,-0.9929219484329224,0.6535818576812744],[0.5740953087806702,0.3640134036540985,0.09609051048755646,0.9359624981880188,-0.9228776097297668,0.6191076636314392,-0.33572638034820557,-0.998464822769165,0.6448632478713989,0.8468661308288574,-0.7557656764984131,0.9527971744537354],[0.7326544523239136,0.5324169397354126,0.9732670187950134,0.7239246964454651,0.2553895115852356,0.8158416152000427,0.665579080581665,0.9287100434303284,-0.5660436153411865,-0.8908745050430298,0.9834234118461609,-0.9981179237365723],[0.9698692560195923,0.7439671754837036,-0.35639333724975586,0.6022987365722656,0.9708116054534912,-0.9278275966644287,-0.9962316155433655,0.834520697593689,0.9714328050613403,0.8158544898033142,0.5902575850486755,0.8199343681335449],[0.9820227026939392,0.9859329462051392,0.5152460932731628,-0.5610517263412476,0.9663666486740112,0.9495157599449158,-0.5204814076423645,0.3104752004146576,0.9859083890914917,0.7797460556030273,0.6738530397415161,0.39197394251823425],[-0.906204104423523,0.11750960350036621,0.8077874779701233,0.416930615901947,0.9829014539718628,0.9902300834655762,0.7847104668617249,0.9945629835128784,0.9868024587631226,-0.26804426312446594,-0.9908867478370667,0.745792806148529],[0.9906190633773804,-0.18231146037578583,0.9757838249206543,0.9986750483512878,0.2544330954551697,-0.954406201839447,0.586924135684967,-0.23537978529930115,0.9550502896308899,0.2551196813583374,0.9929869771003723,0.0905260294675827],[0.9707273244857788,0.6956090927124023,0.6280022263526917,0.7902868390083313,0.9343841671943665,0.9895793795585632,0.9436282515525818,-0.10834990441799164,-0.3431110680103302,0.9986709952354431,0.508673906326294,0.9949509501457214],[0.8283132910728455,0.9432437419891357,0.9491764903068542,0.9995353817939758,0.5712320804595947,0.8055236339569092,0.6781865954399109,0.8272573351860046,0.8314797282218933,0.8778655529022217,0.9944959282875061,0.997386634349823]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0,\"cmin\":-1.0,\"cmax\":1.0},\"title\":{\"text\":\"OV Copying Score for each head in GPT-2 Small\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('b62ac579-268a-4cb8-a4db-de9c144e5855');\n",
+       "var gd = document.getElementById('bac66523-6094-49ba-be5d-8fd8a0f66934');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -2769,10 +2756,10 @@
    "execution_count": 52,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:02:05.721386Z",
-     "iopub.status.busy": "2024-12-14T01:02:05.721038Z",
-     "iopub.status.idle": "2024-12-14T01:02:05.752513Z",
-     "shell.execute_reply": "2024-12-14T01:02:05.751952Z"
+     "iopub.execute_input": "2024-12-31T02:21:22.854964Z",
+     "iopub.status.busy": "2024-12-31T02:21:22.854794Z",
+     "iopub.status.idle": "2024-12-31T02:21:22.884617Z",
+     "shell.execute_reply": "2024-12-31T02:21:22.884185Z"
     }
    },
    "outputs": [
@@ -2783,9 +2770,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"3c8c77e9-6054-4886-85c5-8868e8e7c01c\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"3c8c77e9-6054-4886-85c5-8868e8e7c01c\")) {                    Plotly.newPlot(                        \"3c8c77e9-6054-4886-85c5-8868e8e7c01c\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003eFull OV Copying Score=%{x}\\u003cbr\\u003eOV Copying Score=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"L0H0\",\"L0H1\",\"L0H2\",\"L0H3\",\"L0H4\",\"L0H5\",\"L0H6\",\"L0H7\",\"L0H8\",\"L0H9\",\"L0H10\",\"L0H11\",\"L1H0\",\"L1H1\",\"L1H2\",\"L1H3\",\"L1H4\",\"L1H5\",\"L1H6\",\"L1H7\",\"L1H8\",\"L1H9\",\"L1H10\",\"L1H11\",\"L2H0\",\"L2H1\",\"L2H2\",\"L2H3\",\"L2H4\",\"L2H5\",\"L2H6\",\"L2H7\",\"L2H8\",\"L2H9\",\"L2H10\",\"L2H11\",\"L3H0\",\"L3H1\",\"L3H2\",\"L3H3\",\"L3H4\",\"L3H5\",\"L3H6\",\"L3H7\",\"L3H8\",\"L3H9\",\"L3H10\",\"L3H11\",\"L4H0\",\"L4H1\",\"L4H2\",\"L4H3\",\"L4H4\",\"L4H5\",\"L4H6\",\"L4H7\",\"L4H8\",\"L4H9\",\"L4H10\",\"L4H11\",\"L5H0\",\"L5H1\",\"L5H2\",\"L5H3\",\"L5H4\",\"L5H5\",\"L5H6\",\"L5H7\",\"L5H8\",\"L5H9\",\"L5H10\",\"L5H11\",\"L6H0\",\"L6H1\",\"L6H2\",\"L6H3\",\"L6H4\",\"L6H5\",\"L6H6\",\"L6H7\",\"L6H8\",\"L6H9\",\"L6H10\",\"L6H11\",\"L7H0\",\"L7H1\",\"L7H2\",\"L7H3\",\"L7H4\",\"L7H5\",\"L7H6\",\"L7H7\",\"L7H8\",\"L7H9\",\"L7H10\",\"L7H11\",\"L8H0\",\"L8H1\",\"L8H2\",\"L8H3\",\"L8H4\",\"L8H5\",\"L8H6\",\"L8H7\",\"L8H8\",\"L8H9\",\"L8H10\",\"L8H11\",\"L9H0\",\"L9H1\",\"L9H2\",\"L9H3\",\"L9H4\",\"L9H5\",\"L9H6\",\"L9H7\",\"L9H8\",\"L9H9\",\"L9H10\",\"L9H11\",\"L10H0\",\"L10H1\",\"L10H2\",\"L10H3\",\"L10H4\",\"L10H5\",\"L10H6\",\"L10H7\",\"L10H8\",\"L10H9\",\"L10H10\",\"L10H11\",\"L11H0\",\"L11H1\",\"L11H2\",\"L11H3\",\"L11H4\",\"L11H5\",\"L11H6\",\"L11H7\",\"L11H8\",\"L11H9\",\"L11H10\",\"L11H11\"],\"legendgroup\":\"\",\"marker\":{\"color\":\"#636efa\",\"symbol\":\"circle\"},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[0.8356367349624634,0.5853534936904907,0.5105841159820557,0.7843377590179443,0.8644161224365234,0.7026589512825012,0.8969924449920654,0.5868823528289795,0.42486509680747986,-0.16337518393993378,0.46268585324287415,0.276053786277771,-0.052920110523700714,-0.31773144006729126,-0.4810579717159271,-0.7838066220283508,-0.6360211968421936,-0.77586829662323,0.9681803584098816,0.8119116425514221,-0.7510464787483215,-0.6878445744514465,-0.6429887413978577,-0.9985856413841248,-0.6598325967788696,0.9152501821517944,0.5461500883102417,-0.48743969202041626,0.7720563411712646,-0.7541061639785767,-0.8472450971603394,-0.6948987245559692,-0.1557510495185852,0.2444225549697876,-0.9106622338294983,-0.9439151287078857,0.648689866065979,-0.5592910647392273,0.5935593247413635,0.7843040823936462,-0.8150346279144287,0.613004744052887,0.16785882413387299,0.35195910930633545,-0.6837262511253357,0.2223764955997467,-0.9929219484329224,0.6535818576812744,0.5740953087806702,0.3640134036540985,0.09609051048755646,0.9359624981880188,-0.9228776097297668,0.6191076636314392,-0.33572638034820557,-0.998464822769165,0.6448632478713989,0.8468661308288574,-0.7557656764984131,0.9527971744537354,0.7326544523239136,0.5324169397354126,0.9732670187950134,0.7239246964454651,0.2553895115852356,0.8158416152000427,0.665579080581665,0.9287100434303284,-0.5660436153411865,-0.8908745050430298,0.9834234118461609,-0.9981179237365723,0.9698692560195923,0.7439671754837036,-0.35639333724975586,0.6022987365722656,0.9708116054534912,-0.9278275966644287,-0.9962316155433655,0.834520697593689,0.9714328050613403,0.8158544898033142,0.5902575850486755,0.8199343681335449,0.9820227026939392,0.9859329462051392,0.5152460932731628,-0.5610517263412476,0.9663666486740112,0.9495157599449158,-0.5204814076423645,0.3104752004146576,0.9859083890914917,0.7797460556030273,0.6738530397415161,0.39197394251823425,-0.906204104423523,0.11750960350036621,0.8077874779701233,0.416930615901947,0.9829014539718628,0.9902300834655762,0.7847104668617249,0.9945629835128784,0.9868024587631226,-0.26804426312446594,-0.9908867478370667,0.745792806148529,0.9906190633773804,-0.18231146037578583,0.9757838249206543,0.9986750483512878,0.2544330954551697,-0.954406201839447,0.586924135684967,-0.23537978529930115,0.9550502896308899,0.2551196813583374,0.9929869771003723,0.0905260294675827,0.9707273244857788,0.6956090927124023,0.6280022263526917,0.7902868390083313,0.9343841671943665,0.9895793795585632,0.9436282515525818,-0.10834990441799164,-0.3431110680103302,0.9986709952354431,0.508673906326294,0.9949509501457214,0.8283132910728455,0.9432437419891357,0.9491764903068542,0.9995353817939758,0.5712320804595947,0.8055236339569092,0.6781865954399109,0.8272573351860046,0.8314797282218933,0.8778655529022217,0.9944959282875061,0.997386634349823],\"xaxis\":\"x\",\"y\":[0.7775008678436279,0.35272663831710815,0.25961872935295105,0.6670258045196533,0.838425874710083,0.5584433078765869,0.844474732875824,0.41379114985466003,0.244889497756958,0.02815753035247326,0.3584096431732178,0.16288283467292786,-0.4541913866996765,-0.6529324650764465,-0.5484569668769836,-0.7990368604660034,-0.7736426591873169,-0.8522582650184631,0.977432370185852,0.6626251935958862,-0.7303223609924316,-0.7007019519805908,-0.6946623921394348,-0.9996723532676697,-0.7837162613868713,0.8967758417129517,0.4750956892967224,-0.6671973466873169,0.7881463170051575,-0.8547748923301697,-0.9054183959960938,-0.5749384760856628,-0.321751207113266,-0.0285941194742918,-0.9247617125511169,-0.9699268937110901,0.5864036083221436,-0.76143479347229,0.5971695184707642,0.7854391932487488,-0.8788884878158569,0.3908745348453522,0.0447387658059597,0.1102800965309143,-0.8169987201690674,0.2212953418493271,-0.9939578771591187,0.5774401426315308,0.5254791378974915,0.30490121245384216,-0.10729170590639114,0.9433152675628662,-0.9314427971839905,0.5273631811141968,-0.4264712929725647,-0.9984429478645325,0.5296757817268372,0.8604294657707214,-0.8895052075386047,0.9556970000267029,0.6629188656806946,0.4295697808265686,0.9736858010292053,0.655548095703125,0.12201863527297974,0.7442769408226013,0.5037954449653625,0.95253586769104,-0.6507166624069214,-0.9316278100013733,0.9791510701179504,-0.9972586035728455,0.9613031148910522,0.7501779794692993,-0.3806658983230591,0.6429785490036011,0.9557769298553467,-0.9428837895393372,-0.9948079586029053,0.7852989435195923,0.9657301306724548,0.7073014974594116,0.36872273683547974,0.8128010034561157,0.9659482836723328,0.9730120301246643,0.31900620460510254,-0.30290529131889343,0.9790952801704407,0.9357922673225403,-0.5550314784049988,-0.0054661668837070465,0.986777663230896,0.8249568343162537,0.5664296746253967,0.1000528484582901,-0.9464486837387085,-0.25471991300582886,0.6522327065467834,0.14152583479881287,0.9884141683578491,0.9860584735870361,0.6949271559715271,0.9901811480522156,0.9791203141212463,-0.23595543205738068,-0.982071042060852,0.6506688594818115,0.9895945191383362,-0.291781485080719,0.9714024662971497,0.9951602220535278,0.18783727288246155,-0.9460937976837158,0.4780191481113434,-0.2489192634820938,0.9437099099159241,0.11866225302219391,0.9941242933273315,-0.38088199496269226,0.9564487338066101,0.5542722344398499,0.42118069529533386,0.6628788113594055,0.8659593462944031,0.9937117695808411,0.9069075584411621,0.3981107473373413,-0.4134218096733093,0.9971914887428284,0.3459664583206177,0.9938657283782959,0.5891268253326416,0.9313738942146301,0.9268401861190796,0.9993563890457153,0.6227542161941528,0.8463947772979736,0.6584343910217285,0.8423123955726624,0.2978499233722687,0.8728678822517395,0.9963143467903137,0.9867526292800903],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Full OV Copying Score\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"OV Copying Score\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"OV Copying Score for each head in GPT-2 Small\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"916543d2-faa6-4038-b529-7295899da0ca\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"916543d2-faa6-4038-b529-7295899da0ca\")) {                    Plotly.newPlot(                        \"916543d2-faa6-4038-b529-7295899da0ca\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003eFull OV Copying Score=%{x}\\u003cbr\\u003eOV Copying Score=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"L0H0\",\"L0H1\",\"L0H2\",\"L0H3\",\"L0H4\",\"L0H5\",\"L0H6\",\"L0H7\",\"L0H8\",\"L0H9\",\"L0H10\",\"L0H11\",\"L1H0\",\"L1H1\",\"L1H2\",\"L1H3\",\"L1H4\",\"L1H5\",\"L1H6\",\"L1H7\",\"L1H8\",\"L1H9\",\"L1H10\",\"L1H11\",\"L2H0\",\"L2H1\",\"L2H2\",\"L2H3\",\"L2H4\",\"L2H5\",\"L2H6\",\"L2H7\",\"L2H8\",\"L2H9\",\"L2H10\",\"L2H11\",\"L3H0\",\"L3H1\",\"L3H2\",\"L3H3\",\"L3H4\",\"L3H5\",\"L3H6\",\"L3H7\",\"L3H8\",\"L3H9\",\"L3H10\",\"L3H11\",\"L4H0\",\"L4H1\",\"L4H2\",\"L4H3\",\"L4H4\",\"L4H5\",\"L4H6\",\"L4H7\",\"L4H8\",\"L4H9\",\"L4H10\",\"L4H11\",\"L5H0\",\"L5H1\",\"L5H2\",\"L5H3\",\"L5H4\",\"L5H5\",\"L5H6\",\"L5H7\",\"L5H8\",\"L5H9\",\"L5H10\",\"L5H11\",\"L6H0\",\"L6H1\",\"L6H2\",\"L6H3\",\"L6H4\",\"L6H5\",\"L6H6\",\"L6H7\",\"L6H8\",\"L6H9\",\"L6H10\",\"L6H11\",\"L7H0\",\"L7H1\",\"L7H2\",\"L7H3\",\"L7H4\",\"L7H5\",\"L7H6\",\"L7H7\",\"L7H8\",\"L7H9\",\"L7H10\",\"L7H11\",\"L8H0\",\"L8H1\",\"L8H2\",\"L8H3\",\"L8H4\",\"L8H5\",\"L8H6\",\"L8H7\",\"L8H8\",\"L8H9\",\"L8H10\",\"L8H11\",\"L9H0\",\"L9H1\",\"L9H2\",\"L9H3\",\"L9H4\",\"L9H5\",\"L9H6\",\"L9H7\",\"L9H8\",\"L9H9\",\"L9H10\",\"L9H11\",\"L10H0\",\"L10H1\",\"L10H2\",\"L10H3\",\"L10H4\",\"L10H5\",\"L10H6\",\"L10H7\",\"L10H8\",\"L10H9\",\"L10H10\",\"L10H11\",\"L11H0\",\"L11H1\",\"L11H2\",\"L11H3\",\"L11H4\",\"L11H5\",\"L11H6\",\"L11H7\",\"L11H8\",\"L11H9\",\"L11H10\",\"L11H11\"],\"legendgroup\":\"\",\"marker\":{\"color\":\"#636efa\",\"symbol\":\"circle\"},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[0.8356367349624634,0.5853534936904907,0.5105841159820557,0.7843377590179443,0.8644161224365234,0.7026589512825012,0.8969924449920654,0.5868823528289795,0.42486509680747986,-0.16337518393993378,0.46268585324287415,0.276053786277771,-0.052920110523700714,-0.31773144006729126,-0.4810579717159271,-0.7838066220283508,-0.6360211968421936,-0.77586829662323,0.9681803584098816,0.8119116425514221,-0.7510464787483215,-0.6878445744514465,-0.6429887413978577,-0.9985856413841248,-0.6598325967788696,0.9152501821517944,0.5461500883102417,-0.48743969202041626,0.7720563411712646,-0.7541061639785767,-0.8472450971603394,-0.6948987245559692,-0.1557510495185852,0.2444225549697876,-0.9106622338294983,-0.9439151287078857,0.648689866065979,-0.5592910647392273,0.5935593247413635,0.7843040823936462,-0.8150346279144287,0.613004744052887,0.16785882413387299,0.35195910930633545,-0.6837262511253357,0.2223764955997467,-0.9929219484329224,0.6535818576812744,0.5740953087806702,0.3640134036540985,0.09609051048755646,0.9359624981880188,-0.9228776097297668,0.6191076636314392,-0.33572638034820557,-0.998464822769165,0.6448632478713989,0.8468661308288574,-0.7557656764984131,0.9527971744537354,0.7326544523239136,0.5324169397354126,0.9732670187950134,0.7239246964454651,0.2553895115852356,0.8158416152000427,0.665579080581665,0.9287100434303284,-0.5660436153411865,-0.8908745050430298,0.9834234118461609,-0.9981179237365723,0.9698692560195923,0.7439671754837036,-0.35639333724975586,0.6022987365722656,0.9708116054534912,-0.9278275966644287,-0.9962316155433655,0.834520697593689,0.9714328050613403,0.8158544898033142,0.5902575850486755,0.8199343681335449,0.9820227026939392,0.9859329462051392,0.5152460932731628,-0.5610517263412476,0.9663666486740112,0.9495157599449158,-0.5204814076423645,0.3104752004146576,0.9859083890914917,0.7797460556030273,0.6738530397415161,0.39197394251823425,-0.906204104423523,0.11750960350036621,0.8077874779701233,0.416930615901947,0.9829014539718628,0.9902300834655762,0.7847104668617249,0.9945629835128784,0.9868024587631226,-0.26804426312446594,-0.9908867478370667,0.745792806148529,0.9906190633773804,-0.18231146037578583,0.9757838249206543,0.9986750483512878,0.2544330954551697,-0.954406201839447,0.586924135684967,-0.23537978529930115,0.9550502896308899,0.2551196813583374,0.9929869771003723,0.0905260294675827,0.9707273244857788,0.6956090927124023,0.6280022263526917,0.7902868390083313,0.9343841671943665,0.9895793795585632,0.9436282515525818,-0.10834990441799164,-0.3431110680103302,0.9986709952354431,0.508673906326294,0.9949509501457214,0.8283132910728455,0.9432437419891357,0.9491764903068542,0.9995353817939758,0.5712320804595947,0.8055236339569092,0.6781865954399109,0.8272573351860046,0.8314797282218933,0.8778655529022217,0.9944959282875061,0.997386634349823],\"xaxis\":\"x\",\"y\":[0.7775008678436279,0.35272663831710815,0.25961872935295105,0.6670258045196533,0.838425874710083,0.5584433078765869,0.844474732875824,0.41379114985466003,0.244889497756958,0.02815753035247326,0.3584096431732178,0.16288283467292786,-0.4541913866996765,-0.6529324650764465,-0.5484569668769836,-0.7990368604660034,-0.7736426591873169,-0.8522582650184631,0.977432370185852,0.6626251935958862,-0.7303223609924316,-0.7007019519805908,-0.6946623921394348,-0.9996723532676697,-0.7837162613868713,0.8967758417129517,0.4750956892967224,-0.6671973466873169,0.7881463170051575,-0.8547748923301697,-0.9054183959960938,-0.5749384760856628,-0.321751207113266,-0.0285941194742918,-0.9247617125511169,-0.9699268937110901,0.5864036083221436,-0.76143479347229,0.5971695184707642,0.7854391932487488,-0.8788884878158569,0.3908745348453522,0.0447387658059597,0.1102800965309143,-0.8169987201690674,0.2212953418493271,-0.9939578771591187,0.5774401426315308,0.5254791378974915,0.30490121245384216,-0.10729170590639114,0.9433152675628662,-0.9314427971839905,0.5273631811141968,-0.4264712929725647,-0.9984429478645325,0.5296757817268372,0.8604294657707214,-0.8895052075386047,0.9556970000267029,0.6629188656806946,0.4295697808265686,0.9736858010292053,0.655548095703125,0.12201863527297974,0.7442769408226013,0.5037954449653625,0.95253586769104,-0.6507166624069214,-0.9316278100013733,0.9791510701179504,-0.9972586035728455,0.9613031148910522,0.7501779794692993,-0.3806658983230591,0.6429785490036011,0.9557769298553467,-0.9428837895393372,-0.9948079586029053,0.7852989435195923,0.9657301306724548,0.7073014974594116,0.36872273683547974,0.8128010034561157,0.9659482836723328,0.9730120301246643,0.31900620460510254,-0.30290529131889343,0.9790952801704407,0.9357922673225403,-0.5550314784049988,-0.0054661668837070465,0.986777663230896,0.8249568343162537,0.5664296746253967,0.1000528484582901,-0.9464486837387085,-0.25471991300582886,0.6522327065467834,0.14152583479881287,0.9884141683578491,0.9860584735870361,0.6949271559715271,0.9901811480522156,0.9791203141212463,-0.23595543205738068,-0.982071042060852,0.6506688594818115,0.9895945191383362,-0.291781485080719,0.9714024662971497,0.9951602220535278,0.18783727288246155,-0.9460937976837158,0.4780191481113434,-0.2489192634820938,0.9437099099159241,0.11866225302219391,0.9941242933273315,-0.38088199496269226,0.9564487338066101,0.5542722344398499,0.42118069529533386,0.6628788113594055,0.8659593462944031,0.9937117695808411,0.9069075584411621,0.3981107473373413,-0.4134218096733093,0.9971914887428284,0.3459664583206177,0.9938657283782959,0.5891268253326416,0.9313738942146301,0.9268401861190796,0.9993563890457153,0.6227542161941528,0.8463947772979736,0.6584343910217285,0.8423123955726624,0.2978499233722687,0.8728678822517395,0.9963143467903137,0.9867526292800903],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Full OV Copying Score\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"OV Copying Score\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"OV Copying Score for each head in GPT-2 Small\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('3c8c77e9-6054-4886-85c5-8868e8e7c01c');\n",
+       "var gd = document.getElementById('916543d2-faa6-4038-b529-7295899da0ca');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -2825,10 +2812,10 @@
    "execution_count": 53,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:02:05.754764Z",
-     "iopub.status.busy": "2024-12-14T01:02:05.754419Z",
-     "iopub.status.idle": "2024-12-14T01:02:05.757816Z",
-     "shell.execute_reply": "2024-12-14T01:02:05.757251Z"
+     "iopub.execute_input": "2024-12-31T02:21:22.886585Z",
+     "iopub.status.busy": "2024-12-31T02:21:22.886418Z",
+     "iopub.status.idle": "2024-12-31T02:21:22.889676Z",
+     "shell.execute_reply": "2024-12-31T02:21:22.889250Z"
     }
    },
    "outputs": [
@@ -2869,17 +2856,17 @@
    "execution_count": 54,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:02:05.759822Z",
-     "iopub.status.busy": "2024-12-14T01:02:05.759655Z",
-     "iopub.status.idle": "2024-12-14T01:02:08.062119Z",
-     "shell.execute_reply": "2024-12-14T01:02:08.061525Z"
+     "iopub.execute_input": "2024-12-31T02:21:22.891703Z",
+     "iopub.status.busy": "2024-12-31T02:21:22.891310Z",
+     "iopub.status.idle": "2024-12-31T02:21:25.036501Z",
+     "shell.execute_reply": "2024-12-31T02:21:25.035907Z"
     }
    },
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "ef67630185584553867488aa7abe9fa9",
+       "model_id": "428a330dae074773aad72f7de9f6b219",
        "version_major": 2,
        "version_minor": 0
       },
@@ -2955,10 +2942,10 @@
    "execution_count": 55,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:02:08.064531Z",
-     "iopub.status.busy": "2024-12-14T01:02:08.064357Z",
-     "iopub.status.idle": "2024-12-14T01:02:08.069953Z",
-     "shell.execute_reply": "2024-12-14T01:02:08.069480Z"
+     "iopub.execute_input": "2024-12-31T02:21:25.038945Z",
+     "iopub.status.busy": "2024-12-31T02:21:25.038770Z",
+     "iopub.status.idle": "2024-12-31T02:21:25.044379Z",
+     "shell.execute_reply": "2024-12-31T02:21:25.043814Z"
     }
    },
    "outputs": [],
@@ -3020,10 +3007,10 @@
    "execution_count": 56,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:02:08.071912Z",
-     "iopub.status.busy": "2024-12-14T01:02:08.071742Z",
-     "iopub.status.idle": "2024-12-14T01:02:08.076151Z",
-     "shell.execute_reply": "2024-12-14T01:02:08.075560Z"
+     "iopub.execute_input": "2024-12-31T02:21:25.046510Z",
+     "iopub.status.busy": "2024-12-31T02:21:25.046134Z",
+     "iopub.status.idle": "2024-12-31T02:21:25.050325Z",
+     "shell.execute_reply": "2024-12-31T02:21:25.049883Z"
     }
    },
    "outputs": [
@@ -3062,10 +3049,10 @@
    "execution_count": 57,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:02:08.078214Z",
-     "iopub.status.busy": "2024-12-14T01:02:08.078044Z",
-     "iopub.status.idle": "2024-12-14T01:02:08.082026Z",
-     "shell.execute_reply": "2024-12-14T01:02:08.081479Z"
+     "iopub.execute_input": "2024-12-31T02:21:25.052228Z",
+     "iopub.status.busy": "2024-12-31T02:21:25.052063Z",
+     "iopub.status.idle": "2024-12-31T02:21:25.055994Z",
+     "shell.execute_reply": "2024-12-31T02:21:25.055440Z"
     }
    },
    "outputs": [
@@ -3136,10 +3123,10 @@
    "execution_count": 58,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:02:08.084310Z",
-     "iopub.status.busy": "2024-12-14T01:02:08.083977Z",
-     "iopub.status.idle": "2024-12-14T01:02:08.374838Z",
-     "shell.execute_reply": "2024-12-14T01:02:08.374272Z"
+     "iopub.execute_input": "2024-12-31T02:21:25.057877Z",
+     "iopub.status.busy": "2024-12-31T02:21:25.057714Z",
+     "iopub.status.idle": "2024-12-31T02:21:25.655954Z",
+     "shell.execute_reply": "2024-12-31T02:21:25.655463Z"
     }
    },
    "outputs": [
@@ -3150,9 +3137,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"76053182-4b0b-44d5-9e6f-dc68cfcabb60\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"76053182-4b0b-44d5-9e6f-dc68cfcabb60\")) {                    Plotly.newPlot(                        \"76053182-4b0b-44d5-9e6f-dc68cfcabb60\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162],\"xaxis\":\"x\",\"y\":[262144,2621440,4718592,7077888,9175040,11272192,13631488,15728640,18087936,20185088,22282240,33292288,44302336,55312384,66322432,77332480,88342528,99352576,110362624,121372672,132382720,143392768,154402816,165412864,176422912,187432960,198443008,209453056,220463104,264503296,308281344,352321536,396361728,440401920,484442112,528482304,572522496,616300544,660340736,704380928,748421120,792461312,836501504,880279552,924319744,968359936,1012400128,1056440320,1100480512,1144520704,1188298752,1232338944,1276379136,1320419328,1364459520,1408499712,1452277760,1496317952,1540358144,1584398336,1628438528,1672478720,1716518912,1760296960,1804337152,1848377344,1892417536,1936457728,1980497920,2024275968,2068316160,2112356352,2156396544,2200436736,2420375552,2640314368,2860515328,3080454144,3300392960,3520331776,3740270592,3960471552,4180410368,4400349184,4620288000,4840488960,5060427776,5280366592,5500305408,5720506368,5940445184,6160384000,6380322816,6600523776,6820462592,7040401408,7260340224,7480279040,7700480000,7920418816,8140357632,8360296448,8580497408,8800436224,9020375040,9240313856,9460514816,9680453632,9900392448,10120331264,10340270080,10560471040,10780409856,11000348672,11220287488,11440488448,11660427264,11880366080,12100304896,12320505856,12540444672,12760383488,12980322304,13200523264,13420462080,13640400896,13860339712,14080278528,14300479488,14520418304,14740357120,14960295936,15180496896,15400435712,15620374528,15840313344,16060514304,16280453120,16500391936,16720330752,16940269568,17160470528,17380409344,17600348160,17820286976,18040487936,18260426752,18480365568,18700304384,18920505344,19140444160,19360382976,19580321792,19800522752,20020461568,20240400384,20460339200,20680278016,20900478976,21120417792,21340356608,21560295424,21780496384],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"},\"type\":\"log\"},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Checkpoint Values for attn-only-2l (Log scale)\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"5b380b3b-e660-43ec-a723-02466b223aae\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"5b380b3b-e660-43ec-a723-02466b223aae\")) {                    Plotly.newPlot(                        \"5b380b3b-e660-43ec-a723-02466b223aae\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162],\"xaxis\":\"x\",\"y\":[262144,2621440,4718592,7077888,9175040,11272192,13631488,15728640,18087936,20185088,22282240,33292288,44302336,55312384,66322432,77332480,88342528,99352576,110362624,121372672,132382720,143392768,154402816,165412864,176422912,187432960,198443008,209453056,220463104,264503296,308281344,352321536,396361728,440401920,484442112,528482304,572522496,616300544,660340736,704380928,748421120,792461312,836501504,880279552,924319744,968359936,1012400128,1056440320,1100480512,1144520704,1188298752,1232338944,1276379136,1320419328,1364459520,1408499712,1452277760,1496317952,1540358144,1584398336,1628438528,1672478720,1716518912,1760296960,1804337152,1848377344,1892417536,1936457728,1980497920,2024275968,2068316160,2112356352,2156396544,2200436736,2420375552,2640314368,2860515328,3080454144,3300392960,3520331776,3740270592,3960471552,4180410368,4400349184,4620288000,4840488960,5060427776,5280366592,5500305408,5720506368,5940445184,6160384000,6380322816,6600523776,6820462592,7040401408,7260340224,7480279040,7700480000,7920418816,8140357632,8360296448,8580497408,8800436224,9020375040,9240313856,9460514816,9680453632,9900392448,10120331264,10340270080,10560471040,10780409856,11000348672,11220287488,11440488448,11660427264,11880366080,12100304896,12320505856,12540444672,12760383488,12980322304,13200523264,13420462080,13640400896,13860339712,14080278528,14300479488,14520418304,14740357120,14960295936,15180496896,15400435712,15620374528,15840313344,16060514304,16280453120,16500391936,16720330752,16940269568,17160470528,17380409344,17600348160,17820286976,18040487936,18260426752,18480365568,18700304384,18920505344,19140444160,19360382976,19580321792,19800522752,20020461568,20240400384,20460339200,20680278016,20900478976,21120417792,21340356608,21560295424,21780496384],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"},\"type\":\"log\"},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Checkpoint Values for attn-only-2l (Log scale)\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('76053182-4b0b-44d5-9e6f-dc68cfcabb60');\n",
+       "var gd = document.getElementById('5b380b3b-e660-43ec-a723-02466b223aae');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -3189,9 +3176,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"22829199-cf37-49cf-8817-a42bc6f79448\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"22829199-cf37-49cf-8817-a42bc6f79448\")) {                    Plotly.newPlot(                        \"22829199-cf37-49cf-8817-a42bc6f79448\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162],\"xaxis\":\"x\",\"y\":[196608,3342336,6291456,9240576,12386304,15335424,18284544,21233664,24379392,27328512,30277632,45219840,60358656,75300864,90243072,105381888,120324096,135266304,150208512,165347328,180289536,195231744,210370560,225312768,240254976,255197184,270336000,285278208,300220416,360382464,420347904,480313344,540278784,600244224,660209664,720371712,780337152,840302592,900268032,960233472,1020198912,1080360960,1140326400,1200291840,1260257280,1320222720,1380384768,1440350208,1500315648,1560281088,1620246528,1680211968,1740374016,1800339456,1860304896,1920270336,1980235776,2040201216,2100363264,2160328704,2220294144,2280259584,2340225024,2400387072,2460352512,2520317952,2580283392,2640248832,2700214272,2760376320,2820341760,2880307200,2940272640,3000238080,3300261888,3600285696,3900309504,4200333312,4500357120,4800380928,5100208128,5400231936,5700255744,6000279552,6300303360,6600327168,6900350976,7200374784,7500201984,7800225792,8100249600,8400273408,8700297216,9000321024,9300344832,9600368640,9900392448,10200219648,10500243456,10800267264,11100291072,11400314880,11700338688,12000362496,12300386304,12600213504,12900237312,13200261120,13500284928,13800308736,14100332544,14400356352,14700380160,15000207360,15300231168,15600254976,15900278784,16200302592,16500326400,16800350208,17100374016,17400201216,17700225024,18000248832,18300272640,18600296448,18900320256,19200344064,19500367872,19800391680,20100218880,20400242688,20700266496,21000290304,21300314112,21600337920,21900361728,22200385536,22500212736,22800236544,23100260352,23400284160,23700307968,24000331776,24300355584,24600379392,24900206592,25200230400,25500254208,25800278016,26100301824,26400325632,26700349440,27000373248,27300200448,27600224256,27900248064,28200271872,28500295680,28800319488,29100343296,29400367104,29700390912],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"},\"type\":\"log\"},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Checkpoint Values for solu-12l (Log scale)\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"2e1b7810-7383-4f28-8e08-f69e548f4ae1\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"2e1b7810-7383-4f28-8e08-f69e548f4ae1\")) {                    Plotly.newPlot(                        \"2e1b7810-7383-4f28-8e08-f69e548f4ae1\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162],\"xaxis\":\"x\",\"y\":[196608,3342336,6291456,9240576,12386304,15335424,18284544,21233664,24379392,27328512,30277632,45219840,60358656,75300864,90243072,105381888,120324096,135266304,150208512,165347328,180289536,195231744,210370560,225312768,240254976,255197184,270336000,285278208,300220416,360382464,420347904,480313344,540278784,600244224,660209664,720371712,780337152,840302592,900268032,960233472,1020198912,1080360960,1140326400,1200291840,1260257280,1320222720,1380384768,1440350208,1500315648,1560281088,1620246528,1680211968,1740374016,1800339456,1860304896,1920270336,1980235776,2040201216,2100363264,2160328704,2220294144,2280259584,2340225024,2400387072,2460352512,2520317952,2580283392,2640248832,2700214272,2760376320,2820341760,2880307200,2940272640,3000238080,3300261888,3600285696,3900309504,4200333312,4500357120,4800380928,5100208128,5400231936,5700255744,6000279552,6300303360,6600327168,6900350976,7200374784,7500201984,7800225792,8100249600,8400273408,8700297216,9000321024,9300344832,9600368640,9900392448,10200219648,10500243456,10800267264,11100291072,11400314880,11700338688,12000362496,12300386304,12600213504,12900237312,13200261120,13500284928,13800308736,14100332544,14400356352,14700380160,15000207360,15300231168,15600254976,15900278784,16200302592,16500326400,16800350208,17100374016,17400201216,17700225024,18000248832,18300272640,18600296448,18900320256,19200344064,19500367872,19800391680,20100218880,20400242688,20700266496,21000290304,21300314112,21600337920,21900361728,22200385536,22500212736,22800236544,23100260352,23400284160,23700307968,24000331776,24300355584,24600379392,24900206592,25200230400,25500254208,25800278016,26100301824,26400325632,26700349440,27000373248,27300200448,27600224256,27900248064,28200271872,28500295680,28800319488,29100343296,29400367104,29700390912],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"},\"type\":\"log\"},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Checkpoint Values for solu-12l (Log scale)\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('22829199-cf37-49cf-8817-a42bc6f79448');\n",
+       "var gd = document.getElementById('2e1b7810-7383-4f28-8e08-f69e548f4ae1');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -3228,9 +3215,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"86fd24b0-a205-4e6c-8105-f79fcf57c539\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"86fd24b0-a205-4e6c-8105-f79fcf57c539\")) {                    Plotly.newPlot(                        \"86fd24b0-a205-4e6c-8105-f79fcf57c539\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536,537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603,604,605,606,607,608],\"xaxis\":\"x\",\"y\":[0,10,20,30,40,50,60,70,80,90,100,150,200,250,300,350,400,450,500,550,600,650,700,750,800,850,900,950,1000,1050,1100,1150,1200,1250,1300,1350,1400,1450,1500,1550,1600,1650,1700,1750,1800,1850,1900,1950,2000,2100,2200,2300,2400,2500,2600,2700,2800,2900,3000,3100,3200,3300,3400,3500,3600,3700,3800,3900,4000,4100,4200,4300,4400,4500,4600,4700,4800,4900,5000,5100,5200,5300,5400,5500,5600,5700,5800,5900,6000,6100,6200,6300,6400,6500,6600,6700,6800,6900,7000,7100,7200,7300,7400,7500,7600,7700,7800,7900,8000,8100,8200,8300,8400,8500,8600,8700,8800,8900,9000,9100,9200,9300,9400,9500,9600,9700,9800,9900,10000,10100,10200,10300,10400,10500,10600,10700,10800,10900,11000,11100,11200,11300,11400,11500,11600,11700,11800,11900,12000,12100,12200,12300,12400,12500,12600,12700,12800,12900,13000,13100,13200,13300,13400,13500,13600,13700,13800,13900,14000,14100,14200,14300,14400,14500,14600,14700,14800,14900,15000,15100,15200,15300,15400,15500,15600,15700,15800,15900,16000,16100,16200,16300,16400,16500,16600,16700,16800,16900,17000,17100,17200,17300,17400,17500,17600,17700,17800,17900,18000,18100,18200,18300,18400,18500,18600,18700,18800,18900,19000,19100,19200,19300,19400,19500,19600,19700,19800,19900,20000,21000,22000,23000,24000,25000,26000,27000,28000,29000,30000,31000,32000,33000,34000,35000,36000,37000,38000,39000,40000,41000,42000,43000,44000,45000,46000,47000,48000,49000,50000,51000,52000,53000,54000,55000,56000,57000,58000,59000,60000,61000,62000,63000,64000,65000,66000,67000,68000,69000,70000,71000,72000,73000,74000,75000,76000,77000,78000,79000,80000,81000,82000,83000,84000,85000,86000,87000,88000,89000,90000,91000,92000,93000,94000,95000,96000,97000,98000,99000,100000,101000,102000,103000,104000,105000,106000,107000,108000,109000,110000,111000,112000,113000,114000,115000,116000,117000,118000,119000,120000,121000,122000,123000,124000,125000,126000,127000,128000,129000,130000,131000,132000,133000,134000,135000,136000,137000,138000,139000,140000,141000,142000,143000,144000,145000,146000,147000,148000,149000,150000,151000,152000,153000,154000,155000,156000,157000,158000,159000,160000,161000,162000,163000,164000,165000,166000,167000,168000,169000,170000,171000,172000,173000,174000,175000,176000,177000,178000,179000,180000,181000,182000,183000,184000,185000,186000,187000,188000,189000,190000,191000,192000,193000,194000,195000,196000,197000,198000,199000,200000,201000,202000,203000,204000,205000,206000,207000,208000,209000,210000,211000,212000,213000,214000,215000,216000,217000,218000,219000,220000,221000,222000,223000,224000,225000,226000,227000,228000,229000,230000,231000,232000,233000,234000,235000,236000,237000,238000,239000,240000,241000,242000,243000,244000,245000,246000,247000,248000,249000,250000,251000,252000,253000,254000,255000,256000,257000,258000,259000,260000,261000,262000,263000,264000,265000,266000,267000,268000,269000,270000,271000,272000,273000,274000,275000,276000,277000,278000,279000,280000,281000,282000,283000,284000,285000,286000,287000,288000,289000,290000,291000,292000,293000,294000,295000,296000,297000,298000,299000,300000,301000,302000,303000,304000,305000,306000,307000,308000,309000,310000,311000,312000,313000,314000,315000,316000,317000,318000,319000,320000,321000,322000,323000,324000,325000,326000,327000,328000,329000,330000,331000,332000,333000,334000,335000,336000,337000,338000,339000,340000,341000,342000,343000,344000,345000,346000,347000,348000,349000,350000,351000,352000,353000,354000,355000,356000,357000,358000,359000,360000,361000,362000,363000,364000,365000,366000,367000,368000,369000,370000,371000,372000,373000,374000,375000,376000,377000,378000,379000,380000,381000,382000,383000,384000,385000,386000,387000,388000,389000,390000,391000,392000,393000,394000,395000,396000,397000,398000,399000,400000],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"},\"type\":\"log\"},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Checkpoint Values for stanford-gpt2-small-a (Log scale)\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"81083222-8fc2-4750-982b-6c7df91a2cfc\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"81083222-8fc2-4750-982b-6c7df91a2cfc\")) {                    Plotly.newPlot(                        \"81083222-8fc2-4750-982b-6c7df91a2cfc\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536,537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603,604,605,606,607,608],\"xaxis\":\"x\",\"y\":[0,10,20,30,40,50,60,70,80,90,100,150,200,250,300,350,400,450,500,550,600,650,700,750,800,850,900,950,1000,1050,1100,1150,1200,1250,1300,1350,1400,1450,1500,1550,1600,1650,1700,1750,1800,1850,1900,1950,2000,2100,2200,2300,2400,2500,2600,2700,2800,2900,3000,3100,3200,3300,3400,3500,3600,3700,3800,3900,4000,4100,4200,4300,4400,4500,4600,4700,4800,4900,5000,5100,5200,5300,5400,5500,5600,5700,5800,5900,6000,6100,6200,6300,6400,6500,6600,6700,6800,6900,7000,7100,7200,7300,7400,7500,7600,7700,7800,7900,8000,8100,8200,8300,8400,8500,8600,8700,8800,8900,9000,9100,9200,9300,9400,9500,9600,9700,9800,9900,10000,10100,10200,10300,10400,10500,10600,10700,10800,10900,11000,11100,11200,11300,11400,11500,11600,11700,11800,11900,12000,12100,12200,12300,12400,12500,12600,12700,12800,12900,13000,13100,13200,13300,13400,13500,13600,13700,13800,13900,14000,14100,14200,14300,14400,14500,14600,14700,14800,14900,15000,15100,15200,15300,15400,15500,15600,15700,15800,15900,16000,16100,16200,16300,16400,16500,16600,16700,16800,16900,17000,17100,17200,17300,17400,17500,17600,17700,17800,17900,18000,18100,18200,18300,18400,18500,18600,18700,18800,18900,19000,19100,19200,19300,19400,19500,19600,19700,19800,19900,20000,21000,22000,23000,24000,25000,26000,27000,28000,29000,30000,31000,32000,33000,34000,35000,36000,37000,38000,39000,40000,41000,42000,43000,44000,45000,46000,47000,48000,49000,50000,51000,52000,53000,54000,55000,56000,57000,58000,59000,60000,61000,62000,63000,64000,65000,66000,67000,68000,69000,70000,71000,72000,73000,74000,75000,76000,77000,78000,79000,80000,81000,82000,83000,84000,85000,86000,87000,88000,89000,90000,91000,92000,93000,94000,95000,96000,97000,98000,99000,100000,101000,102000,103000,104000,105000,106000,107000,108000,109000,110000,111000,112000,113000,114000,115000,116000,117000,118000,119000,120000,121000,122000,123000,124000,125000,126000,127000,128000,129000,130000,131000,132000,133000,134000,135000,136000,137000,138000,139000,140000,141000,142000,143000,144000,145000,146000,147000,148000,149000,150000,151000,152000,153000,154000,155000,156000,157000,158000,159000,160000,161000,162000,163000,164000,165000,166000,167000,168000,169000,170000,171000,172000,173000,174000,175000,176000,177000,178000,179000,180000,181000,182000,183000,184000,185000,186000,187000,188000,189000,190000,191000,192000,193000,194000,195000,196000,197000,198000,199000,200000,201000,202000,203000,204000,205000,206000,207000,208000,209000,210000,211000,212000,213000,214000,215000,216000,217000,218000,219000,220000,221000,222000,223000,224000,225000,226000,227000,228000,229000,230000,231000,232000,233000,234000,235000,236000,237000,238000,239000,240000,241000,242000,243000,244000,245000,246000,247000,248000,249000,250000,251000,252000,253000,254000,255000,256000,257000,258000,259000,260000,261000,262000,263000,264000,265000,266000,267000,268000,269000,270000,271000,272000,273000,274000,275000,276000,277000,278000,279000,280000,281000,282000,283000,284000,285000,286000,287000,288000,289000,290000,291000,292000,293000,294000,295000,296000,297000,298000,299000,300000,301000,302000,303000,304000,305000,306000,307000,308000,309000,310000,311000,312000,313000,314000,315000,316000,317000,318000,319000,320000,321000,322000,323000,324000,325000,326000,327000,328000,329000,330000,331000,332000,333000,334000,335000,336000,337000,338000,339000,340000,341000,342000,343000,344000,345000,346000,347000,348000,349000,350000,351000,352000,353000,354000,355000,356000,357000,358000,359000,360000,361000,362000,363000,364000,365000,366000,367000,368000,369000,370000,371000,372000,373000,374000,375000,376000,377000,378000,379000,380000,381000,382000,383000,384000,385000,386000,387000,388000,389000,390000,391000,392000,393000,394000,395000,396000,397000,398000,399000,400000],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"},\"type\":\"log\"},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Checkpoint Values for stanford-gpt2-small-a (Log scale)\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('86fd24b0-a205-4e6c-8105-f79fcf57c539');\n",
+       "var gd = document.getElementById('81083222-8fc2-4750-982b-6c7df91a2cfc');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -3267,9 +3254,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"4a6e81dc-ddca-460f-873f-0a213188ab89\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"4a6e81dc-ddca-460f-873f-0a213188ab89\")) {                    Plotly.newPlot(                        \"4a6e81dc-ddca-460f-873f-0a213188ab89\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49],\"xaxis\":\"x\",\"y\":[832,1664,2496,3328,4160,4992,5824,6656,7488,8320,9152,9984,10816,11648,12480,13312,14144,14976,15808,16640,17472,18304,19136,19968,20800,21632,22464,23296,24128,24960,25792,26624,27456,28288,29120,29952,30784,31616,32448,33280,34112,34944,35776,36608,37440,38272,39104,39936,40768,41600],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"}},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Checkpoint Values for solu-1l-pile (Linear scale)\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"f160c230-ab5d-46fa-be53-58d80e50a0e7\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"f160c230-ab5d-46fa-be53-58d80e50a0e7\")) {                    Plotly.newPlot(                        \"f160c230-ab5d-46fa-be53-58d80e50a0e7\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49],\"xaxis\":\"x\",\"y\":[832,1664,2496,3328,4160,4992,5824,6656,7488,8320,9152,9984,10816,11648,12480,13312,14144,14976,15808,16640,17472,18304,19136,19968,20800,21632,22464,23296,24128,24960,25792,26624,27456,28288,29120,29952,30784,31616,32448,33280,34112,34944,35776,36608,37440,38272,39104,39936,40768,41600],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"}},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Checkpoint Values for solu-1l-pile (Linear scale)\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('4a6e81dc-ddca-460f-873f-0a213188ab89');\n",
+       "var gd = document.getElementById('f160c230-ab5d-46fa-be53-58d80e50a0e7');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -3306,9 +3293,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"e2877398-410c-47b5-9f61-1fd97414d652\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"e2877398-410c-47b5-9f61-1fd97414d652\")) {                    Plotly.newPlot(                        \"e2877398-410c-47b5-9f61-1fd97414d652\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99],\"xaxis\":\"x\",\"y\":[326,652,978,1304,1630,1956,2282,2608,2934,3260,3586,3912,4238,4564,4890,5216,5542,5868,6194,6520,6846,7172,7498,7824,8150,8476,8802,9128,9454,9780,10106,10432,10758,11084,11410,11736,12062,12388,12714,13040,13366,13692,14018,14344,14670,14996,15322,15648,15974,16300,16626,16952,17278,17604,17930,18256,18582,18908,19234,19560,19886,20212,20538,20864,21190,21516,21842,22168,22494,22820,23146,23472,23798,24124,24450,24776,25102,25428,25754,26080,26406,26732,27058,27384,27710,28036,28362,28688,29014,29340,29666,29992,30318,30644,30970,31296,31622,31948,32274,32600],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"}},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Checkpoint Values for solu-6l-pile (Linear scale)\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"77531c7d-147c-41c7-88a0-f58896e92710\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"77531c7d-147c-41c7-88a0-f58896e92710\")) {                    Plotly.newPlot(                        \"77531c7d-147c-41c7-88a0-f58896e92710\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99],\"xaxis\":\"x\",\"y\":[326,652,978,1304,1630,1956,2282,2608,2934,3260,3586,3912,4238,4564,4890,5216,5542,5868,6194,6520,6846,7172,7498,7824,8150,8476,8802,9128,9454,9780,10106,10432,10758,11084,11410,11736,12062,12388,12714,13040,13366,13692,14018,14344,14670,14996,15322,15648,15974,16300,16626,16952,17278,17604,17930,18256,18582,18908,19234,19560,19886,20212,20538,20864,21190,21516,21842,22168,22494,22820,23146,23472,23798,24124,24450,24776,25102,25428,25754,26080,26406,26732,27058,27384,27710,28036,28362,28688,29014,29340,29666,29992,30318,30644,30970,31296,31622,31948,32274,32600],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"}},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Checkpoint Values for solu-6l-pile (Linear scale)\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('e2877398-410c-47b5-9f61-1fd97414d652');\n",
+       "var gd = document.getElementById('77531c7d-147c-41c7-88a0-f58896e92710');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -3383,10 +3370,10 @@
    "execution_count": 59,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:02:08.377335Z",
-     "iopub.status.busy": "2024-12-14T01:02:08.376955Z",
-     "iopub.status.idle": "2024-12-14T01:02:08.380056Z",
-     "shell.execute_reply": "2024-12-14T01:02:08.379541Z"
+     "iopub.execute_input": "2024-12-31T02:21:25.658035Z",
+     "iopub.status.busy": "2024-12-31T02:21:25.657861Z",
+     "iopub.status.idle": "2024-12-31T02:21:25.660967Z",
+     "shell.execute_reply": "2024-12-31T02:21:25.660534Z"
     }
    },
    "outputs": [],
@@ -3413,10 +3400,10 @@
    "execution_count": 60,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:02:08.382096Z",
-     "iopub.status.busy": "2024-12-14T01:02:08.381792Z",
-     "iopub.status.idle": "2024-12-14T01:02:08.385346Z",
-     "shell.execute_reply": "2024-12-14T01:02:08.384774Z"
+     "iopub.execute_input": "2024-12-31T02:21:25.662985Z",
+     "iopub.status.busy": "2024-12-31T02:21:25.662690Z",
+     "iopub.status.idle": "2024-12-31T02:21:25.665998Z",
+     "shell.execute_reply": "2024-12-31T02:21:25.665554Z"
     }
    },
    "outputs": [],
@@ -3448,10 +3435,10 @@
    "execution_count": 61,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-12-14T01:02:08.387531Z",
-     "iopub.status.busy": "2024-12-14T01:02:08.387061Z",
-     "iopub.status.idle": "2024-12-14T01:02:08.419341Z",
-     "shell.execute_reply": "2024-12-14T01:02:08.418749Z"
+     "iopub.execute_input": "2024-12-31T02:21:25.667996Z",
+     "iopub.status.busy": "2024-12-31T02:21:25.667607Z",
+     "iopub.status.idle": "2024-12-31T02:21:25.697601Z",
+     "shell.execute_reply": "2024-12-31T02:21:25.696939Z"
     }
    },
    "outputs": [
@@ -3462,9 +3449,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"e8c48c58-6618-4eb6-8790-24b50ffe4409\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"e8c48c58-6618-4eb6-8790-24b50ffe4409\")) {                    Plotly.newPlot(                        \"e8c48c58-6618-4eb6-8790-24b50ffe4409\",                        [],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"},\"type\":\"log\"},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Induction Loss over training: solu-2l\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"1dda3792-cb5b-475a-95ed-9a9f74c8cb4a\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"1dda3792-cb5b-475a-95ed-9a9f74c8cb4a\")) {                    Plotly.newPlot(                        \"1dda3792-cb5b-475a-95ed-9a9f74c8cb4a\",                        [],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"},\"type\":\"log\"},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Induction Loss over training: solu-2l\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('e8c48c58-6618-4eb6-8790-24b50ffe4409');\n",
+       "var gd = document.getElementById('1dda3792-cb5b-475a-95ed-9a9f74c8cb4a');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -3526,106 +3513,7 @@
   "widgets": {
    "application/vnd.jupyter.widget-state+json": {
     "state": {
-     "0093c35dd59c47adaed249b404d1a0b1": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_3adaa7786f684ff6abb501929b91bfeb",
-       "placeholder": "​",
-       "style": "IPY_MODEL_236a3fe34dbb46b9a945a6c32dc4ed37",
-       "tabbable": null,
-       "tooltip": null,
-       "value": " 124/124 [00:00&lt;00:00, 21.8kB/s]"
-      }
-     },
-     "0347d987e2964dd4a31348434040c0c7": {
-      "model_module": "@jupyter-widgets/base",
-      "model_module_version": "2.0.0",
-      "model_name": "LayoutModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/base",
-       "_model_module_version": "2.0.0",
-       "_model_name": "LayoutModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "LayoutView",
-       "align_content": null,
-       "align_items": null,
-       "align_self": null,
-       "border_bottom": null,
-       "border_left": null,
-       "border_right": null,
-       "border_top": null,
-       "bottom": null,
-       "display": null,
-       "flex": null,
-       "flex_flow": null,
-       "grid_area": null,
-       "grid_auto_columns": null,
-       "grid_auto_flow": null,
-       "grid_auto_rows": null,
-       "grid_column": null,
-       "grid_gap": null,
-       "grid_row": null,
-       "grid_template_areas": null,
-       "grid_template_columns": null,
-       "grid_template_rows": null,
-       "height": null,
-       "justify_content": null,
-       "justify_items": null,
-       "left": null,
-       "margin": null,
-       "max_height": null,
-       "max_width": null,
-       "min_height": null,
-       "min_width": null,
-       "object_fit": null,
-       "object_position": null,
-       "order": null,
-       "overflow": null,
-       "padding": null,
-       "right": null,
-       "top": null,
-       "visibility": null,
-       "width": null
-      }
-     },
-     "0f1f32e4ccb44e77b4a1aab830f601a5": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_9fce92e16f574800b2a9add4b6c10ef8",
-       "placeholder": "​",
-       "style": "IPY_MODEL_bfdbeaa99bc942abb6a71e94629483ab",
-       "tabbable": null,
-       "tooltip": null,
-       "value": "tokenizer.json: 100%"
-      }
-     },
-     "16df306b166340878aae8ee0d7d2af94": {
+     "00a1e307bf234324ab8c80e2b11eeeee": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "ProgressStyleModel",
@@ -3641,7 +3529,7 @@
        "description_width": ""
       }
      },
-     "19da29c1b3cf4fae9710cd70f0baf271": {
+     "01f4e058fc8a4ad68133f91434605cb6": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -3694,31 +3582,7 @@
        "width": null
       }
      },
-     "1bfe4dc8f9d64ed3a70a0e6e41b61d34": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HBoxModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HBoxModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HBoxView",
-       "box_style": "",
-       "children": [
-        "IPY_MODEL_d671580b566f41a3b09b9e31d0a97129",
-        "IPY_MODEL_f50d3b1ce11a44d8b5801bdd32a86fa8",
-        "IPY_MODEL_a3a46ce06ec04834810c4ab96c959684"
-       ],
-       "layout": "IPY_MODEL_7129d439a9d549d48802b08586b8c24f",
-       "tabbable": null,
-       "tooltip": null
-      }
-     },
-     "1c150b40a42e403b848a74fd683a4788": {
+     "03d7575b9cbc48da96ff17baa48be4c4": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -3771,7 +3635,7 @@
        "width": null
       }
      },
-     "1dfb46a04fe14273b07f0621a7a25096": {
+     "046865f8c6fa4333a2aafa4067066f4e": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLModel",
@@ -3786,15 +3650,15 @@
        "_view_name": "HTMLView",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_33b3f2ed76434c13b68809069903b693",
+       "layout": "IPY_MODEL_f650970bbec24731978e61f5639f83d8",
        "placeholder": "​",
-       "style": "IPY_MODEL_ae9c1384a8ef40a9b2a32844f1646716",
+       "style": "IPY_MODEL_359e2768fce646a0a615612fee3e508e",
        "tabbable": null,
        "tooltip": null,
-       "value": "100%"
+       "value": " 12/12 [00:19&lt;00:00,  1.58s/it]"
       }
      },
-     "217459b8733641fda6a21621dc147583": {
+     "07913cfb30d7470ea6fe716d3599cbfd": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -3847,7 +3711,7 @@
        "width": null
       }
      },
-     "22fa2830a0ab4ae4bb190db8ab08f59c": {
+     "11156915bf414366b67a6a0f9e6c1960": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -3900,25 +3764,31 @@
        "width": null
       }
      },
-     "236a3fe34dbb46b9a945a6c32dc4ed37": {
+     "16f9e028019c47c19107395141087d9c": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
+      "model_name": "HBoxModel",
       "state": {
+       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
+       "_model_name": "HBoxModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
+       "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
+       "_view_name": "HBoxView",
+       "box_style": "",
+       "children": [
+        "IPY_MODEL_26bec98fe7ff4d0f8c9f2668af368199",
+        "IPY_MODEL_214306a3558d400bb2e91903307496dc",
+        "IPY_MODEL_57e1eddf9db444169ab9759f877f7de7"
+       ],
+       "layout": "IPY_MODEL_95093d79632c42e89f65648356e52288",
+       "tabbable": null,
+       "tooltip": null
       }
      },
-     "25f38d8b724d481397c21ffd4e2ae408": {
+     "18993102a6b540c6b4b3a8e65a3285a5": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -3971,108 +3841,64 @@
        "width": null
       }
      },
-     "2974465a897b4ac9ba7d573f8e1ea5f8": {
-      "model_module": "@jupyter-widgets/base",
+     "18bf6e516c5442bfb281ac041eb68673": {
+      "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "LayoutModel",
+      "model_name": "ProgressStyleModel",
       "state": {
-       "_model_module": "@jupyter-widgets/base",
+       "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "LayoutModel",
+       "_model_name": "ProgressStyleModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
-       "_view_name": "LayoutView",
-       "align_content": null,
-       "align_items": null,
-       "align_self": null,
-       "border_bottom": null,
-       "border_left": null,
-       "border_right": null,
-       "border_top": null,
-       "bottom": null,
-       "display": null,
-       "flex": null,
-       "flex_flow": null,
-       "grid_area": null,
-       "grid_auto_columns": null,
-       "grid_auto_flow": null,
-       "grid_auto_rows": null,
-       "grid_column": null,
-       "grid_gap": null,
-       "grid_row": null,
-       "grid_template_areas": null,
-       "grid_template_columns": null,
-       "grid_template_rows": null,
-       "height": null,
-       "justify_content": null,
-       "justify_items": null,
-       "left": null,
-       "margin": null,
-       "max_height": null,
-       "max_width": null,
-       "min_height": null,
-       "min_width": null,
-       "object_fit": null,
-       "object_position": null,
-       "order": null,
-       "overflow": null,
-       "padding": null,
-       "right": null,
-       "top": null,
-       "visibility": null,
-       "width": null
+       "_view_name": "StyleView",
+       "bar_color": null,
+       "description_width": ""
       }
      },
-     "2b653bed330e47bfaf1eb97604ffb148": {
+     "1bcf54ed3c7c4a20937255e48eb68170": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HBoxModel",
+      "model_name": "HTMLStyleModel",
       "state": {
-       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HBoxModel",
+       "_model_name": "HTMLStyleModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
+       "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
-       "_view_name": "HBoxView",
-       "box_style": "",
-       "children": [
-        "IPY_MODEL_c70847ad8e5543748ffbab29446b51c1",
-        "IPY_MODEL_4116100d429b4b9faf99f919b8cba2fd",
-        "IPY_MODEL_6304f591778f49058f9403f77c821c56"
-       ],
-       "layout": "IPY_MODEL_1c150b40a42e403b848a74fd683a4788",
-       "tabbable": null,
-       "tooltip": null
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
       }
      },
-     "2f95bda34e124bdb8ad42b97968c2560": {
+     "1e93a9f0f574448abab30220598d14ea": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HBoxModel",
+      "model_name": "HTMLModel",
       "state": {
        "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HBoxModel",
+       "_model_name": "HTMLModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "HBoxView",
-       "box_style": "",
-       "children": [
-        "IPY_MODEL_3a73a2ce3d684e57af5ead4d970bb546",
-        "IPY_MODEL_45071b25ff23496094a139022c8edc54",
-        "IPY_MODEL_0093c35dd59c47adaed249b404d1a0b1"
-       ],
-       "layout": "IPY_MODEL_2fbd1d8558174aaeae9046da37a2dc08",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_07913cfb30d7470ea6fe716d3599cbfd",
+       "placeholder": "​",
+       "style": "IPY_MODEL_d7e88b54c04d415fac710f14830d32b3",
        "tabbable": null,
-       "tooltip": null
+       "tooltip": null,
+       "value": "100%"
       }
      },
-     "2fbd1d8558174aaeae9046da37a2dc08": {
+     "1f0204d5124744658fb76eb12e63d0a4": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -4125,362 +3951,106 @@
        "width": null
       }
      },
-     "30e033f4a4a745f99416fa6c47e6b1b8": {
+     "214306a3558d400bb2e91903307496dc": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
+      "model_name": "FloatProgressModel",
       "state": {
+       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
+       "_model_name": "FloatProgressModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
+       "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
+       "_view_name": "ProgressView",
+       "bar_style": "success",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_f8b7bd5b73384893a9643f6f0f3bc7a5",
+       "max": 352824413.0,
+       "min": 0.0,
+       "orientation": "horizontal",
+       "style": "IPY_MODEL_00a1e307bf234324ab8c80e2b11eeeee",
+       "tabbable": null,
+       "tooltip": null,
+       "value": 352824413.0
       }
      },
-     "3383a9989d6a4375bccecc436f7df6c0": {
-      "model_module": "@jupyter-widgets/base",
+     "26bec98fe7ff4d0f8c9f2668af368199": {
+      "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "LayoutModel",
+      "model_name": "HTMLModel",
       "state": {
-       "_model_module": "@jupyter-widgets/base",
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "LayoutModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "LayoutView",
-       "align_content": null,
-       "align_items": null,
-       "align_self": null,
-       "border_bottom": null,
-       "border_left": null,
-       "border_right": null,
-       "border_top": null,
-       "bottom": null,
-       "display": null,
-       "flex": null,
-       "flex_flow": null,
-       "grid_area": null,
-       "grid_auto_columns": null,
-       "grid_auto_flow": null,
-       "grid_auto_rows": null,
-       "grid_column": null,
-       "grid_gap": null,
-       "grid_row": null,
-       "grid_template_areas": null,
-       "grid_template_columns": null,
-       "grid_template_rows": null,
-       "height": null,
-       "justify_content": null,
-       "justify_items": null,
-       "left": null,
-       "margin": null,
-       "max_height": null,
-       "max_width": null,
-       "min_height": null,
-       "min_width": null,
-       "object_fit": null,
-       "object_position": null,
-       "order": null,
-       "overflow": null,
-       "padding": null,
-       "right": null,
-       "top": null,
-       "visibility": null,
-       "width": null
-      }
-     },
-     "33b3f2ed76434c13b68809069903b693": {
-      "model_module": "@jupyter-widgets/base",
-      "model_module_version": "2.0.0",
-      "model_name": "LayoutModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/base",
-       "_model_module_version": "2.0.0",
-       "_model_name": "LayoutModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "LayoutView",
-       "align_content": null,
-       "align_items": null,
-       "align_self": null,
-       "border_bottom": null,
-       "border_left": null,
-       "border_right": null,
-       "border_top": null,
-       "bottom": null,
-       "display": null,
-       "flex": null,
-       "flex_flow": null,
-       "grid_area": null,
-       "grid_auto_columns": null,
-       "grid_auto_flow": null,
-       "grid_auto_rows": null,
-       "grid_column": null,
-       "grid_gap": null,
-       "grid_row": null,
-       "grid_template_areas": null,
-       "grid_template_columns": null,
-       "grid_template_rows": null,
-       "height": null,
-       "justify_content": null,
-       "justify_items": null,
-       "left": null,
-       "margin": null,
-       "max_height": null,
-       "max_width": null,
-       "min_height": null,
-       "min_width": null,
-       "object_fit": null,
-       "object_position": null,
-       "order": null,
-       "overflow": null,
-       "padding": null,
-       "right": null,
-       "top": null,
-       "visibility": null,
-       "width": null
-      }
-     },
-     "355529cac414494c83e338e54c92a4cc": {
-      "model_module": "@jupyter-widgets/base",
-      "model_module_version": "2.0.0",
-      "model_name": "LayoutModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/base",
-       "_model_module_version": "2.0.0",
-       "_model_name": "LayoutModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "LayoutView",
-       "align_content": null,
-       "align_items": null,
-       "align_self": null,
-       "border_bottom": null,
-       "border_left": null,
-       "border_right": null,
-       "border_top": null,
-       "bottom": null,
-       "display": null,
-       "flex": null,
-       "flex_flow": null,
-       "grid_area": null,
-       "grid_auto_columns": null,
-       "grid_auto_flow": null,
-       "grid_auto_rows": null,
-       "grid_column": null,
-       "grid_gap": null,
-       "grid_row": null,
-       "grid_template_areas": null,
-       "grid_template_columns": null,
-       "grid_template_rows": null,
-       "height": null,
-       "justify_content": null,
-       "justify_items": null,
-       "left": null,
-       "margin": null,
-       "max_height": null,
-       "max_width": null,
-       "min_height": null,
-       "min_width": null,
-       "object_fit": null,
-       "object_position": null,
-       "order": null,
-       "overflow": null,
-       "padding": null,
-       "right": null,
-       "top": null,
-       "visibility": null,
-       "width": null
-      }
-     },
-     "363a53f62bad40f38d9b57415fcdd27a": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HBoxModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HBoxModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HBoxView",
-       "box_style": "",
-       "children": [
-        "IPY_MODEL_98db25d1775c4574b9702ae9d27f9d76",
-        "IPY_MODEL_975c4858e139485f8f0aa84771e564a4",
-        "IPY_MODEL_3b38cd8e7a4749a49d9442a541dd6d69"
-       ],
-       "layout": "IPY_MODEL_a6d329232ced49958c43f07f02862115",
-       "tabbable": null,
-       "tooltip": null
-      }
-     },
-     "38bdea7d9b64452ea8420690243965cf": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
+       "_model_name": "HTMLModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
        "_view_name": "HTMLView",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_e7fe152978d2459c9cf82d8f9f257479",
+       "layout": "IPY_MODEL_18993102a6b540c6b4b3a8e65a3285a5",
        "placeholder": "​",
-       "style": "IPY_MODEL_a6544264d0d741aa81b06d6cf1050e84",
+       "style": "IPY_MODEL_c420d4cb979d4cb6a77edaf29e09f965",
        "tabbable": null,
        "tooltip": null,
-       "value": " 50/50 [00:02&lt;00:00, 22.25it/s]"
+       "value": "model.safetensors: 100%"
       }
      },
-     "3a73a2ce3d684e57af5ead4d970bb546": {
+     "29b6d4431334417e9690be62ac184ff3": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
+      "model_name": "HBoxModel",
       "state": {
        "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
+       "_model_name": "HBoxModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_19da29c1b3cf4fae9710cd70f0baf271",
-       "placeholder": "​",
-       "style": "IPY_MODEL_30e033f4a4a745f99416fa6c47e6b1b8",
+       "_view_name": "HBoxView",
+       "box_style": "",
+       "children": [
+        "IPY_MODEL_53e1b0c15a7b45b88c0377ead2d99333",
+        "IPY_MODEL_c46d177181494ee1a7dd6e5c9c8c86fa",
+        "IPY_MODEL_e9b4b7ca03a34748a25a569fa3dacc27"
+       ],
+       "layout": "IPY_MODEL_faa2abf5d66c4dfe96ce5fec6486692e",
        "tabbable": null,
-       "tooltip": null,
-       "value": "generation_config.json: 100%"
-      }
-     },
-     "3adaa7786f684ff6abb501929b91bfeb": {
-      "model_module": "@jupyter-widgets/base",
-      "model_module_version": "2.0.0",
-      "model_name": "LayoutModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/base",
-       "_model_module_version": "2.0.0",
-       "_model_name": "LayoutModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "LayoutView",
-       "align_content": null,
-       "align_items": null,
-       "align_self": null,
-       "border_bottom": null,
-       "border_left": null,
-       "border_right": null,
-       "border_top": null,
-       "bottom": null,
-       "display": null,
-       "flex": null,
-       "flex_flow": null,
-       "grid_area": null,
-       "grid_auto_columns": null,
-       "grid_auto_flow": null,
-       "grid_auto_rows": null,
-       "grid_column": null,
-       "grid_gap": null,
-       "grid_row": null,
-       "grid_template_areas": null,
-       "grid_template_columns": null,
-       "grid_template_rows": null,
-       "height": null,
-       "justify_content": null,
-       "justify_items": null,
-       "left": null,
-       "margin": null,
-       "max_height": null,
-       "max_width": null,
-       "min_height": null,
-       "min_width": null,
-       "object_fit": null,
-       "object_position": null,
-       "order": null,
-       "overflow": null,
-       "padding": null,
-       "right": null,
-       "top": null,
-       "visibility": null,
-       "width": null
+       "tooltip": null
       }
      },
-     "3b38cd8e7a4749a49d9442a541dd6d69": {
+     "2c3822722a26406f9a3cfce27fa2f9e2": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
+      "model_name": "FloatProgressModel",
       "state": {
        "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
+       "_model_name": "FloatProgressModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
+       "_view_name": "ProgressView",
+       "bar_style": "success",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_355529cac414494c83e338e54c92a4cc",
-       "placeholder": "​",
-       "style": "IPY_MODEL_72e1c6506989403db938acb8a05caa9a",
+       "layout": "IPY_MODEL_bec7b50387f340908a1ce664d0ee7e46",
+       "max": 26.0,
+       "min": 0.0,
+       "orientation": "horizontal",
+       "style": "IPY_MODEL_762ae9d728004d92a034a51cea8fec66",
        "tabbable": null,
        "tooltip": null,
-       "value": " 1.04M/1.04M [00:00&lt;00:00, 17.5MB/s]"
-      }
-     },
-     "4025e90abd624e92ad601372fe03f72a": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
-      }
-     },
-     "4045d70e18ef41f892b496c2457200e8": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
+       "value": 26.0
       }
      },
-     "40bb01d6e86d499a80c048f392bcbe29": {
+     "30ad307a0269480eb94b844695317e07": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLStyleModel",
@@ -4498,59 +4068,46 @@
        "text_color": null
       }
      },
-     "4116100d429b4b9faf99f919b8cba2fd": {
+     "333364115678404d89398e4c1e998ef3": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "FloatProgressModel",
+      "model_name": "ProgressStyleModel",
       "state": {
-       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "FloatProgressModel",
+       "_model_name": "ProgressStyleModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
+       "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
-       "_view_name": "ProgressView",
-       "bar_style": "success",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_ce2ad39062644901b02faaefcf98b910",
-       "max": 352824413.0,
-       "min": 0.0,
-       "orientation": "horizontal",
-       "style": "IPY_MODEL_16df306b166340878aae8ee0d7d2af94",
-       "tabbable": null,
-       "tooltip": null,
-       "value": 352824413.0
+       "_view_name": "StyleView",
+       "bar_color": null,
+       "description_width": ""
       }
      },
-     "45071b25ff23496094a139022c8edc54": {
+     "33f927be3355434587a90f29e15b30a1": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "FloatProgressModel",
+      "model_name": "HTMLModel",
       "state": {
        "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "FloatProgressModel",
+       "_model_name": "HTMLModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "ProgressView",
-       "bar_style": "success",
+       "_view_name": "HTMLView",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_6e974e0f6e104aee841709766cbfe512",
-       "max": 124.0,
-       "min": 0.0,
-       "orientation": "horizontal",
-       "style": "IPY_MODEL_4025e90abd624e92ad601372fe03f72a",
+       "layout": "IPY_MODEL_01f4e058fc8a4ad68133f91434605cb6",
+       "placeholder": "​",
+       "style": "IPY_MODEL_a099ae8a89304ee0b2e630e61700dd39",
        "tabbable": null,
        "tooltip": null,
-       "value": 124.0
+       "value": "generation_config.json: 100%"
       }
      },
-     "459712eff9964201b95e3cb31857e8da": {
+     "359e2768fce646a0a615612fee3e508e": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLStyleModel",
@@ -4568,7 +4125,7 @@
        "text_color": null
       }
      },
-     "46833033a8eb479ea4c5c87af7d997bb": {
+     "391ab10e327244fa91d8ab82410c81ac": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -4621,7 +4178,23 @@
        "width": null
       }
      },
-     "47b00b0a700a470cba9e0970646bf5a3": {
+     "3bbe6453f4b7453ca7567b066697a38f": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "ProgressStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "ProgressStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "bar_color": null,
+       "description_width": ""
+      }
+     },
+     "411ab9797b3a44b4b5751b6df3a35e03": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -4674,7 +4247,75 @@
        "width": null
       }
      },
-     "485b5446c8864360a4f063c6992e772a": {
+     "419c4221831f458d9160b21cdf5adb9d": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "FloatProgressModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "FloatProgressModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "ProgressView",
+       "bar_style": "success",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_d333a600da514d26af421e9c420ff241",
+       "max": 456318.0,
+       "min": 0.0,
+       "orientation": "horizontal",
+       "style": "IPY_MODEL_3bbe6453f4b7453ca7567b066697a38f",
+       "tabbable": null,
+       "tooltip": null,
+       "value": 456318.0
+      }
+     },
+     "4289a5d73d3241199ee38b1ef4ac475d": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "428a330dae074773aad72f7de9f6b219": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HBoxModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HBoxModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HBoxView",
+       "box_style": "",
+       "children": [
+        "IPY_MODEL_f65616c888c944d3a7778e70fcccebf0",
+        "IPY_MODEL_9491a41bb00f49c9ac6243e9b050e8ea",
+        "IPY_MODEL_fca8d2f8306640ba9f2e11c6d492392e"
+       ],
+       "layout": "IPY_MODEL_11156915bf414366b67a6a0f9e6c1960",
+       "tabbable": null,
+       "tooltip": null
+      }
+     },
+     "4d7242e8d4ec4e1db36416d52dedc096": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -4727,142 +4368,233 @@
        "width": null
       }
      },
-     "4d16a5f6738f41918a9be1a5c6911d80": {
+     "4f5a31f406bc4c8582c7878892bede4f": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
+      "model_name": "FloatProgressModel",
       "state": {
        "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
+       "_model_name": "FloatProgressModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
+       "_view_name": "ProgressView",
+       "bar_style": "success",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_c6ee4432596b4ff999fe1bddd0d46240",
-       "placeholder": "​",
-       "style": "IPY_MODEL_697afa5733c6440f907ee2c1ffb2584e",
+       "layout": "IPY_MODEL_fa2989d4907f487fb94956f67090d1bd",
+       "max": 12.0,
+       "min": 0.0,
+       "orientation": "horizontal",
+       "style": "IPY_MODEL_8c32daa5d0a64fcdaadad700b50b2e27",
        "tabbable": null,
        "tooltip": null,
-       "value": " 456k/456k [00:00&lt;00:00, 35.6MB/s]"
+       "value": 12.0
       }
      },
-     "4ec86ba49fc04f1ea410f2a37bf7ffae": {
+     "53e1b0c15a7b45b88c0377ead2d99333": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "FloatProgressModel",
+      "model_name": "HTMLModel",
       "state": {
        "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "FloatProgressModel",
+       "_model_name": "HTMLModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "ProgressView",
-       "bar_style": "success",
+       "_view_name": "HTMLView",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_b49e96fd18bc45e981e58cefbd348841",
-       "max": 456318.0,
-       "min": 0.0,
-       "orientation": "horizontal",
-       "style": "IPY_MODEL_4045d70e18ef41f892b496c2457200e8",
+       "layout": "IPY_MODEL_a3a591fc7c61495d86702fe435581acf",
+       "placeholder": "​",
+       "style": "IPY_MODEL_9b2871c6328f44629e38d82e383140ac",
        "tabbable": null,
        "tooltip": null,
-       "value": 456318.0
+       "value": "tokenizer.json: 100%"
       }
      },
-     "4f9bedf73f97497db82c1b58ff5117bf": {
+     "57e1eddf9db444169ab9759f877f7de7": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
+      "model_name": "HTMLModel",
       "state": {
+       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
+       "_model_name": "HTMLModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
+       "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_411ab9797b3a44b4b5751b6df3a35e03",
+       "placeholder": "​",
+       "style": "IPY_MODEL_bd64574c041b44d38017dfd4820b2e4b",
+       "tabbable": null,
+       "tooltip": null,
+       "value": " 353M/353M [00:01&lt;00:00, 215MB/s]"
       }
      },
-     "546e22db36d04372a0aa187adbcf0262": {
+     "59a6916e36b44608987b585683320e55": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
+      "model_name": "HBoxModel",
       "state": {
+       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
+       "_model_name": "HBoxModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
+       "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
+       "_view_name": "HBoxView",
+       "box_style": "",
+       "children": [
+        "IPY_MODEL_dc7c542bbf5d4b6ca58dcec68b91e68c",
+        "IPY_MODEL_b1c430d051414eecaccd61435c0ed34c",
+        "IPY_MODEL_af82e06c82bf452fa1a9d0ff9dd9f131"
+       ],
+       "layout": "IPY_MODEL_1f0204d5124744658fb76eb12e63d0a4",
+       "tabbable": null,
+       "tooltip": null
       }
      },
-     "58a2ce5740e543bdaa92d598b1b084dc": {
-      "model_module": "@jupyter-widgets/controls",
+     "699d9373f7ba45d490db037be91b05c5": {
+      "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
+      "model_name": "LayoutModel",
       "state": {
-       "_model_module": "@jupyter-widgets/controls",
+       "_model_module": "@jupyter-widgets/base",
        "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
+       "_model_name": "LayoutModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
+       "_view_name": "LayoutView",
+       "align_content": null,
+       "align_items": null,
+       "align_self": null,
+       "border_bottom": null,
+       "border_left": null,
+       "border_right": null,
+       "border_top": null,
+       "bottom": null,
+       "display": null,
+       "flex": null,
+       "flex_flow": null,
+       "grid_area": null,
+       "grid_auto_columns": null,
+       "grid_auto_flow": null,
+       "grid_auto_rows": null,
+       "grid_column": null,
+       "grid_gap": null,
+       "grid_row": null,
+       "grid_template_areas": null,
+       "grid_template_columns": null,
+       "grid_template_rows": null,
+       "height": null,
+       "justify_content": null,
+       "justify_items": null,
+       "left": null,
+       "margin": null,
+       "max_height": null,
+       "max_width": null,
+       "min_height": null,
+       "min_width": null,
+       "object_fit": null,
+       "object_position": null,
+       "order": null,
+       "overflow": null,
+       "padding": null,
+       "right": null,
+       "top": null,
+       "visibility": null,
+       "width": null
       }
      },
-     "5ad68d8e26e44e76a786ef4b57a2263d": {
+     "6a66981f155d4bf280913b398c75f705": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
+      "model_name": "HBoxModel",
       "state": {
+       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
+       "_model_name": "HBoxModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
+       "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
+       "_view_name": "HBoxView",
+       "box_style": "",
+       "children": [
+        "IPY_MODEL_704a2185ba1e43c18f631342bab60f7f",
+        "IPY_MODEL_2c3822722a26406f9a3cfce27fa2f9e2",
+        "IPY_MODEL_c42f7e12651d47c899dc8ae3cd954dd1"
+       ],
+       "layout": "IPY_MODEL_391ab10e327244fa91d8ab82410c81ac",
+       "tabbable": null,
+       "tooltip": null
       }
      },
-     "5fbc42bb13f64054a6dbbd8c58ecfc59": {
-      "model_module": "@jupyter-widgets/controls",
+     "6f40fa1bcbbf4455817daaefafb9f5b6": {
+      "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
+      "model_name": "LayoutModel",
       "state": {
-       "_model_module": "@jupyter-widgets/controls",
+       "_model_module": "@jupyter-widgets/base",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
+       "_model_name": "LayoutModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
+       "_view_name": "LayoutView",
+       "align_content": null,
+       "align_items": null,
+       "align_self": null,
+       "border_bottom": null,
+       "border_left": null,
+       "border_right": null,
+       "border_top": null,
+       "bottom": null,
+       "display": null,
+       "flex": null,
+       "flex_flow": null,
+       "grid_area": null,
+       "grid_auto_columns": null,
+       "grid_auto_flow": null,
+       "grid_auto_rows": null,
+       "grid_column": null,
+       "grid_gap": null,
+       "grid_row": null,
+       "grid_template_areas": null,
+       "grid_template_columns": null,
+       "grid_template_rows": null,
+       "height": null,
+       "justify_content": null,
+       "justify_items": null,
+       "left": null,
+       "margin": null,
+       "max_height": null,
+       "max_width": null,
+       "min_height": null,
+       "min_width": null,
+       "object_fit": null,
+       "object_position": null,
+       "order": null,
+       "overflow": null,
+       "padding": null,
+       "right": null,
+       "top": null,
+       "visibility": null,
+       "width": null
       }
      },
-     "6304f591778f49058f9403f77c821c56": {
+     "704a2185ba1e43c18f631342bab60f7f": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLModel",
@@ -4877,59 +4609,31 @@
        "_view_name": "HTMLView",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_aa07cd1a18be4feda9dfaae9577809b5",
+       "layout": "IPY_MODEL_ecb2ff817ce746eb85ba29b7cf393758",
        "placeholder": "​",
-       "style": "IPY_MODEL_5fbc42bb13f64054a6dbbd8c58ecfc59",
+       "style": "IPY_MODEL_1bcf54ed3c7c4a20937255e48eb68170",
        "tabbable": null,
        "tooltip": null,
-       "value": " 353M/353M [00:01&lt;00:00, 198MB/s]"
+       "value": "tokenizer_config.json: 100%"
       }
      },
-     "697afa5733c6440f907ee2c1ffb2584e": {
+     "762ae9d728004d92a034a51cea8fec66": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
+      "model_name": "ProgressStyleModel",
       "state": {
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
+       "_model_name": "ProgressStyleModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
        "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
-      }
-     },
-     "69da672452a94ea68b438b3be68f1a24": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "FloatProgressModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "FloatProgressModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "ProgressView",
-       "bar_style": "success",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_0347d987e2964dd4a31348434040c0c7",
-       "max": 50.0,
-       "min": 0.0,
-       "orientation": "horizontal",
-       "style": "IPY_MODEL_858356c58e784bc4b5778ed67747a0d1",
-       "tabbable": null,
-       "tooltip": null,
-       "value": 50.0
+       "bar_color": null,
+       "description_width": ""
       }
      },
-     "6e974e0f6e104aee841709766cbfe512": {
+     "77d883cf087b45fc9a7fc9786dadbe63": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -4982,30 +4686,33 @@
        "width": null
       }
      },
-     "70759237fca04d3084c2a5df2a30f271": {
+     "7c5cfa385cd8435ca3d7703e8106d131": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
+      "model_name": "FloatProgressModel",
       "state": {
        "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
+       "_model_name": "FloatProgressModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
+       "_view_name": "ProgressView",
+       "bar_style": "success",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_47b00b0a700a470cba9e0970646bf5a3",
-       "placeholder": "​",
-       "style": "IPY_MODEL_459712eff9964201b95e3cb31857e8da",
+       "layout": "IPY_MODEL_e4a3d32f101647e0b844ff2ee427907a",
+       "max": 124.0,
+       "min": 0.0,
+       "orientation": "horizontal",
+       "style": "IPY_MODEL_18bf6e516c5442bfb281ac041eb68673",
        "tabbable": null,
        "tooltip": null,
-       "value": " 1.36M/1.36M [00:00&lt;00:00, 9.38MB/s]"
+       "value": 124.0
       }
      },
-     "7129d439a9d549d48802b08586b8c24f": {
+     "82787d08d40948da9d57fa80681d6c01": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -5058,25 +4765,60 @@
        "width": null
       }
      },
-     "72e1c6506989403db938acb8a05caa9a": {
-      "model_module": "@jupyter-widgets/controls",
+     "82af6bbb20854ff98eedf4e5944fad63": {
+      "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
+      "model_name": "LayoutModel",
       "state": {
-       "_model_module": "@jupyter-widgets/controls",
+       "_model_module": "@jupyter-widgets/base",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
+       "_model_name": "LayoutModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
+       "_view_name": "LayoutView",
+       "align_content": null,
+       "align_items": null,
+       "align_self": null,
+       "border_bottom": null,
+       "border_left": null,
+       "border_right": null,
+       "border_top": null,
+       "bottom": null,
+       "display": null,
+       "flex": null,
+       "flex_flow": null,
+       "grid_area": null,
+       "grid_auto_columns": null,
+       "grid_auto_flow": null,
+       "grid_auto_rows": null,
+       "grid_column": null,
+       "grid_gap": null,
+       "grid_row": null,
+       "grid_template_areas": null,
+       "grid_template_columns": null,
+       "grid_template_rows": null,
+       "height": null,
+       "justify_content": null,
+       "justify_items": null,
+       "left": null,
+       "margin": null,
+       "max_height": null,
+       "max_width": null,
+       "min_height": null,
+       "min_width": null,
+       "object_fit": null,
+       "object_position": null,
+       "order": null,
+       "overflow": null,
+       "padding": null,
+       "right": null,
+       "top": null,
+       "visibility": null,
+       "width": null
       }
      },
-     "76995f1052794da19381d92e27c1de96": {
+     "85c96ee49b1e46fdafef7b1f934ef9ad": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -5129,7 +4871,7 @@
        "width": null
       }
      },
-     "7f7ad53041c6499e9178b752e3f0bf66": {
+     "877233dda9cb4d00864bf6470ae98ceb": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -5182,7 +4924,25 @@
        "width": null
       }
      },
-     "8480a1acfe1a4929becf5c1eeb2df5f1": {
+     "895aa0507aab41399395c24c2da2a903": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "8a3b44f2c5d84e94a2bb4637a12748b9": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLModel",
@@ -5197,15 +4957,31 @@
        "_view_name": "HTMLView",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_e4da68381e90404db807b341d42c96a8",
+       "layout": "IPY_MODEL_917ca83d84b043db937ff9901f09976a",
        "placeholder": "​",
-       "style": "IPY_MODEL_4f9bedf73f97497db82c1b58ff5117bf",
+       "style": "IPY_MODEL_bbcadaf9fda64c49bb69a0f9ee183beb",
        "tabbable": null,
        "tooltip": null,
-       "value": "100%"
+       "value": "merges.txt: 100%"
+      }
+     },
+     "8c32daa5d0a64fcdaadad700b50b2e27": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "ProgressStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "ProgressStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "bar_color": null,
+       "description_width": ""
       }
      },
-     "85239cb8d6ba4d288b001290e919d816": {
+     "8c41c6eefd75443fa69c9327b4c1ec8e": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HBoxModel",
@@ -5220,58 +4996,87 @@
        "_view_name": "HBoxView",
        "box_style": "",
        "children": [
-        "IPY_MODEL_0f1f32e4ccb44e77b4a1aab830f601a5",
-        "IPY_MODEL_db49cd92e84f4f20b4d44fe5829790df",
-        "IPY_MODEL_70759237fca04d3084c2a5df2a30f271"
+        "IPY_MODEL_33f927be3355434587a90f29e15b30a1",
+        "IPY_MODEL_7c5cfa385cd8435ca3d7703e8106d131",
+        "IPY_MODEL_e648af8d85b046298af44b614c30d687"
        ],
-       "layout": "IPY_MODEL_c666ac6d4533412e922fc4b9a1084fc1",
+       "layout": "IPY_MODEL_877233dda9cb4d00864bf6470ae98ceb",
        "tabbable": null,
        "tooltip": null
       }
      },
-     "858356c58e784bc4b5778ed67747a0d1": {
+     "8dafe502a1d947ed89517420f0e8d1bd": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
+      "model_name": "HTMLStyleModel",
       "state": {
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
+       "_model_name": "HTMLStyleModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
        "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
       }
      },
-     "975c4858e139485f8f0aa84771e564a4": {
-      "model_module": "@jupyter-widgets/controls",
+     "917ca83d84b043db937ff9901f09976a": {
+      "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
-      "model_name": "FloatProgressModel",
+      "model_name": "LayoutModel",
       "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
+       "_model_module": "@jupyter-widgets/base",
        "_model_module_version": "2.0.0",
-       "_model_name": "FloatProgressModel",
+       "_model_name": "LayoutModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
+       "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
-       "_view_name": "ProgressView",
-       "bar_style": "success",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_25f38d8b724d481397c21ffd4e2ae408",
-       "max": 1042301.0,
-       "min": 0.0,
-       "orientation": "horizontal",
-       "style": "IPY_MODEL_546e22db36d04372a0aa187adbcf0262",
-       "tabbable": null,
-       "tooltip": null,
-       "value": 1042301.0
+       "_view_name": "LayoutView",
+       "align_content": null,
+       "align_items": null,
+       "align_self": null,
+       "border_bottom": null,
+       "border_left": null,
+       "border_right": null,
+       "border_top": null,
+       "bottom": null,
+       "display": null,
+       "flex": null,
+       "flex_flow": null,
+       "grid_area": null,
+       "grid_auto_columns": null,
+       "grid_auto_flow": null,
+       "grid_auto_rows": null,
+       "grid_column": null,
+       "grid_gap": null,
+       "grid_row": null,
+       "grid_template_areas": null,
+       "grid_template_columns": null,
+       "grid_template_rows": null,
+       "height": null,
+       "justify_content": null,
+       "justify_items": null,
+       "left": null,
+       "margin": null,
+       "max_height": null,
+       "max_width": null,
+       "min_height": null,
+       "min_width": null,
+       "object_fit": null,
+       "object_position": null,
+       "order": null,
+       "overflow": null,
+       "padding": null,
+       "right": null,
+       "top": null,
+       "visibility": null,
+       "width": null
       }
      },
-     "977babee536f42158a1d81b4b7aa208d": {
+     "9491a41bb00f49c9ac6243e9b050e8ea": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "FloatProgressModel",
@@ -5287,40 +5092,17 @@
        "bar_style": "success",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_2974465a897b4ac9ba7d573f8e1ea5f8",
-       "max": 12.0,
+       "layout": "IPY_MODEL_949dd78c7d124e8ab4696ed732b1de5c",
+       "max": 50.0,
        "min": 0.0,
        "orientation": "horizontal",
-       "style": "IPY_MODEL_58a2ce5740e543bdaa92d598b1b084dc",
-       "tabbable": null,
-       "tooltip": null,
-       "value": 12.0
-      }
-     },
-     "98db25d1775c4574b9702ae9d27f9d76": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_bc420542d1d34e14978dcc6d9d0471e3",
-       "placeholder": "​",
-       "style": "IPY_MODEL_c741d615fd45485b899d67710062ed85",
+       "style": "IPY_MODEL_333364115678404d89398e4c1e998ef3",
        "tabbable": null,
        "tooltip": null,
-       "value": "vocab.json: 100%"
+       "value": 50.0
       }
      },
-     "9fce92e16f574800b2a9add4b6c10ef8": {
+     "949dd78c7d124e8ab4696ed732b1de5c": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -5373,46 +5155,60 @@
        "width": null
       }
      },
-     "a05caff4cdcb42e8813ccaf3072ea092": {
-      "model_module": "@jupyter-widgets/controls",
+     "95093d79632c42e89f65648356e52288": {
+      "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
+      "model_name": "LayoutModel",
       "state": {
-       "_model_module": "@jupyter-widgets/controls",
+       "_model_module": "@jupyter-widgets/base",
        "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
+       "_model_name": "LayoutModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
-      }
-     },
-     "a3a46ce06ec04834810c4ab96c959684": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_a95e7f64a7d240529cacf7a5a9e1c06b",
-       "placeholder": "​",
-       "style": "IPY_MODEL_40bb01d6e86d499a80c048f392bcbe29",
-       "tabbable": null,
-       "tooltip": null,
-       "value": " 26.0/26.0 [00:00&lt;00:00, 5.03kB/s]"
+       "_view_name": "LayoutView",
+       "align_content": null,
+       "align_items": null,
+       "align_self": null,
+       "border_bottom": null,
+       "border_left": null,
+       "border_right": null,
+       "border_top": null,
+       "bottom": null,
+       "display": null,
+       "flex": null,
+       "flex_flow": null,
+       "grid_area": null,
+       "grid_auto_columns": null,
+       "grid_auto_flow": null,
+       "grid_auto_rows": null,
+       "grid_column": null,
+       "grid_gap": null,
+       "grid_row": null,
+       "grid_template_areas": null,
+       "grid_template_columns": null,
+       "grid_template_rows": null,
+       "height": null,
+       "justify_content": null,
+       "justify_items": null,
+       "left": null,
+       "margin": null,
+       "max_height": null,
+       "max_width": null,
+       "min_height": null,
+       "min_width": null,
+       "object_fit": null,
+       "object_position": null,
+       "order": null,
+       "overflow": null,
+       "padding": null,
+       "right": null,
+       "top": null,
+       "visibility": null,
+       "width": null
       }
      },
-     "a3feec14402a46afa537597c59f70647": {
+     "9a813b30c933431685c2e34827469d49": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLStyleModel",
@@ -5430,7 +5226,7 @@
        "text_color": null
       }
      },
-     "a6544264d0d741aa81b06d6cf1050e84": {
+     "9b2871c6328f44629e38d82e383140ac": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLStyleModel",
@@ -5448,7 +5244,7 @@
        "text_color": null
       }
      },
-     "a6d329232ced49958c43f07f02862115": {
+     "9b98600e30e241658a16044231674040": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -5501,7 +5297,25 @@
        "width": null
       }
      },
-     "a95e7f64a7d240529cacf7a5a9e1c06b": {
+     "a099ae8a89304ee0b2e630e61700dd39": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "a3a591fc7c61495d86702fe435581acf": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -5554,7 +5368,7 @@
        "width": null
       }
      },
-     "aa07cd1a18be4feda9dfaae9577809b5": {
+     "a92a6cd4defc4e12ba9b9ff94af36e62": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -5607,64 +5421,56 @@
        "width": null
       }
      },
-     "ab0021bee3b94de0965c25c38a7962f2": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
-      }
-     },
-     "ae9c1384a8ef40a9b2a32844f1646716": {
+     "af82e06c82bf452fa1a9d0ff9dd9f131": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
+      "model_name": "HTMLModel",
       "state": {
+       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
+       "_model_name": "HTMLModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
+       "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_699d9373f7ba45d490db037be91b05c5",
+       "placeholder": "​",
+       "style": "IPY_MODEL_f5ed129096c34e70bf382ba956e836bf",
+       "tabbable": null,
+       "tooltip": null,
+       "value": " 1.04M/1.04M [00:00&lt;00:00, 5.17MB/s]"
       }
      },
-     "aec5d70e86d84aef8fe50515973443a8": {
+     "b1c430d051414eecaccd61435c0ed34c": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
+      "model_name": "FloatProgressModel",
       "state": {
        "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
+       "_model_name": "FloatProgressModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
+       "_view_name": "ProgressView",
+       "bar_style": "success",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_46833033a8eb479ea4c5c87af7d997bb",
-       "placeholder": "​",
-       "style": "IPY_MODEL_5ad68d8e26e44e76a786ef4b57a2263d",
+       "layout": "IPY_MODEL_77d883cf087b45fc9a7fc9786dadbe63",
+       "max": 1042301.0,
+       "min": 0.0,
+       "orientation": "horizontal",
+       "style": "IPY_MODEL_b995e0b517794c4a935c99155b7bd3c7",
        "tabbable": null,
        "tooltip": null,
-       "value": "merges.txt: 100%"
+       "value": 1042301.0
       }
      },
-     "b49e96fd18bc45e981e58cefbd348841": {
+     "b696d0501fbf4e0f8ca8123a5d9359d8": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -5717,31 +5523,101 @@
        "width": null
       }
      },
-     "baa802cac1bf4a77bd2d6f2da4e0c206": {
+     "b7f610cc1c1d418594f22fd76ac0e072": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "b995e0b517794c4a935c99155b7bd3c7": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "ProgressStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "ProgressStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "bar_color": null,
+       "description_width": ""
+      }
+     },
+     "bb02e41e51664a96b3c06abd21683006": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HBoxModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HBoxModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HBoxView",
+       "box_style": "",
+       "children": [
+        "IPY_MODEL_1e93a9f0f574448abab30220598d14ea",
+        "IPY_MODEL_4f5a31f406bc4c8582c7878892bede4f",
+        "IPY_MODEL_046865f8c6fa4333a2aafa4067066f4e"
+       ],
+       "layout": "IPY_MODEL_4d7242e8d4ec4e1db36416d52dedc096",
+       "tabbable": null,
+       "tooltip": null
+      }
+     },
+     "bbcadaf9fda64c49bb69a0f9ee183beb": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "bd64574c041b44d38017dfd4820b2e4b": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HBoxModel",
+      "model_name": "HTMLStyleModel",
       "state": {
-       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HBoxModel",
+       "_model_name": "HTMLStyleModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
+       "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
-       "_view_name": "HBoxView",
-       "box_style": "",
-       "children": [
-        "IPY_MODEL_aec5d70e86d84aef8fe50515973443a8",
-        "IPY_MODEL_4ec86ba49fc04f1ea410f2a37bf7ffae",
-        "IPY_MODEL_4d16a5f6738f41918a9be1a5c6911d80"
-       ],
-       "layout": "IPY_MODEL_485b5446c8864360a4f063c6992e772a",
-       "tabbable": null,
-       "tooltip": null
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
       }
      },
-     "bc420542d1d34e14978dcc6d9d0471e3": {
+     "bec7b50387f340908a1ce664d0ee7e46": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -5794,7 +5670,7 @@
        "width": null
       }
      },
-     "bd751bbba24c4154acc75b88e0f3a87a": {
+     "c420d4cb979d4cb6a77edaf29e09f965": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLStyleModel",
@@ -5812,25 +5688,56 @@
        "text_color": null
       }
      },
-     "bfdbeaa99bc942abb6a71e94629483ab": {
+     "c42f7e12651d47c899dc8ae3cd954dd1": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
+      "model_name": "HTMLModel",
       "state": {
+       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
+       "_model_name": "HTMLModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
+       "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_6f40fa1bcbbf4455817daaefafb9f5b6",
+       "placeholder": "​",
+       "style": "IPY_MODEL_4289a5d73d3241199ee38b1ef4ac475d",
+       "tabbable": null,
+       "tooltip": null,
+       "value": " 26.0/26.0 [00:00&lt;00:00, 4.36kB/s]"
+      }
+     },
+     "c46d177181494ee1a7dd6e5c9c8c86fa": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "FloatProgressModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "FloatProgressModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "ProgressView",
+       "bar_style": "success",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_9b98600e30e241658a16044231674040",
+       "max": 1355256.0,
+       "min": 0.0,
+       "orientation": "horizontal",
+       "style": "IPY_MODEL_e123c944e91c4d53a6040ffeab016e69",
+       "tabbable": null,
+       "tooltip": null,
+       "value": 1355256.0
       }
      },
-     "c666ac6d4533412e922fc4b9a1084fc1": {
+     "cf524e5a513e435cab49c816b198a542": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -5883,7 +5790,7 @@
        "width": null
       }
      },
-     "c6ee4432596b4ff999fe1bddd0d46240": {
+     "d333a600da514d26af421e9c420ff241": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -5936,7 +5843,43 @@
        "width": null
       }
      },
-     "c70847ad8e5543748ffbab29446b51c1": {
+     "d7e88b54c04d415fac710f14830d32b3": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "dc3cea0e6cea4325879ccc3a197abc5f": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "dc7c542bbf5d4b6ca58dcec68b91e68c": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLModel",
@@ -5951,33 +5894,31 @@
        "_view_name": "HTMLView",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_7f7ad53041c6499e9178b752e3f0bf66",
+       "layout": "IPY_MODEL_b696d0501fbf4e0f8ca8123a5d9359d8",
        "placeholder": "​",
-       "style": "IPY_MODEL_a3feec14402a46afa537597c59f70647",
+       "style": "IPY_MODEL_8dafe502a1d947ed89517420f0e8d1bd",
        "tabbable": null,
        "tooltip": null,
-       "value": "model.safetensors: 100%"
+       "value": "vocab.json: 100%"
       }
      },
-     "c741d615fd45485b899d67710062ed85": {
+     "e123c944e91c4d53a6040ffeab016e69": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
+      "model_name": "ProgressStyleModel",
       "state": {
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
+       "_model_name": "ProgressStyleModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
        "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
+       "bar_color": null,
+       "description_width": ""
       }
      },
-     "ce2ad39062644901b02faaefcf98b910": {
+     "e4a3d32f101647e0b844ff2ee427907a": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -6030,7 +5971,7 @@
        "width": null
       }
      },
-     "d671580b566f41a3b09b9e31d0a97129": {
+     "e648af8d85b046298af44b614c30d687": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLModel",
@@ -6045,41 +5986,91 @@
        "_view_name": "HTMLView",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_f825f61e07764da39007e09a988d2df0",
+       "layout": "IPY_MODEL_82af6bbb20854ff98eedf4e5944fad63",
        "placeholder": "​",
-       "style": "IPY_MODEL_dee4a33c12ba4787ad99f56355b782af",
+       "style": "IPY_MODEL_b7f610cc1c1d418594f22fd76ac0e072",
        "tabbable": null,
        "tooltip": null,
-       "value": "tokenizer_config.json: 100%"
+       "value": " 124/124 [00:00&lt;00:00, 24.2kB/s]"
       }
      },
-     "db49cd92e84f4f20b4d44fe5829790df": {
+     "e9b4b7ca03a34748a25a569fa3dacc27": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "FloatProgressModel",
+      "model_name": "HTMLModel",
       "state": {
        "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "FloatProgressModel",
+       "_model_name": "HTMLModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "ProgressView",
-       "bar_style": "success",
+       "_view_name": "HTMLView",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_f5bf41cb59ed4db4b184f40d99bfbc1a",
-       "max": 1355256.0,
-       "min": 0.0,
-       "orientation": "horizontal",
-       "style": "IPY_MODEL_ab0021bee3b94de0965c25c38a7962f2",
+       "layout": "IPY_MODEL_cf524e5a513e435cab49c816b198a542",
+       "placeholder": "​",
+       "style": "IPY_MODEL_9a813b30c933431685c2e34827469d49",
        "tabbable": null,
        "tooltip": null,
-       "value": 1355256.0
+       "value": " 1.36M/1.36M [00:00&lt;00:00, 6.86MB/s]"
+      }
+     },
+     "ecb2ff817ce746eb85ba29b7cf393758": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "2.0.0",
+      "model_name": "LayoutModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/base",
+       "_model_module_version": "2.0.0",
+       "_model_name": "LayoutModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "LayoutView",
+       "align_content": null,
+       "align_items": null,
+       "align_self": null,
+       "border_bottom": null,
+       "border_left": null,
+       "border_right": null,
+       "border_top": null,
+       "bottom": null,
+       "display": null,
+       "flex": null,
+       "flex_flow": null,
+       "grid_area": null,
+       "grid_auto_columns": null,
+       "grid_auto_flow": null,
+       "grid_auto_rows": null,
+       "grid_column": null,
+       "grid_gap": null,
+       "grid_row": null,
+       "grid_template_areas": null,
+       "grid_template_columns": null,
+       "grid_template_rows": null,
+       "height": null,
+       "justify_content": null,
+       "justify_items": null,
+       "left": null,
+       "margin": null,
+       "max_height": null,
+       "max_width": null,
+       "min_height": null,
+       "min_width": null,
+       "object_fit": null,
+       "object_position": null,
+       "order": null,
+       "overflow": null,
+       "padding": null,
+       "right": null,
+       "top": null,
+       "visibility": null,
+       "width": null
       }
      },
-     "ddd91d21a45e49a8a90fd6693fc0fe90": {
+     "f462bf0c8aef4d9e9fd54629b8fe4434": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HBoxModel",
@@ -6094,16 +6085,16 @@
        "_view_name": "HBoxView",
        "box_style": "",
        "children": [
-        "IPY_MODEL_8480a1acfe1a4929becf5c1eeb2df5f1",
-        "IPY_MODEL_977babee536f42158a1d81b4b7aa208d",
-        "IPY_MODEL_fdf5d6542de342d08a2d9a0482eda1d7"
+        "IPY_MODEL_8a3b44f2c5d84e94a2bb4637a12748b9",
+        "IPY_MODEL_419c4221831f458d9160b21cdf5adb9d",
+        "IPY_MODEL_ffd00559da434d5ea3a744e89e8ab013"
        ],
-       "layout": "IPY_MODEL_76995f1052794da19381d92e27c1de96",
+       "layout": "IPY_MODEL_03d7575b9cbc48da96ff17baa48be4c4",
        "tabbable": null,
        "tooltip": null
       }
      },
-     "dee4a33c12ba4787ad99f56355b782af": {
+     "f5ed129096c34e70bf382ba956e836bf": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLStyleModel",
@@ -6121,7 +6112,7 @@
        "text_color": null
       }
      },
-     "e4da68381e90404db807b341d42c96a8": {
+     "f650970bbec24731978e61f5639f83d8": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -6174,7 +6165,30 @@
        "width": null
       }
      },
-     "e7fe152978d2459c9cf82d8f9f257479": {
+     "f65616c888c944d3a7778e70fcccebf0": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_a92a6cd4defc4e12ba9b9ff94af36e62",
+       "placeholder": "​",
+       "style": "IPY_MODEL_dc3cea0e6cea4325879ccc3a197abc5f",
+       "tabbable": null,
+       "tooltip": null,
+       "value": "100%"
+      }
+     },
+     "f8b7bd5b73384893a9643f6f0f3bc7a5": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -6227,57 +6241,7 @@
        "width": null
       }
      },
-     "ef67630185584553867488aa7abe9fa9": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HBoxModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HBoxModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HBoxView",
-       "box_style": "",
-       "children": [
-        "IPY_MODEL_1dfb46a04fe14273b07f0621a7a25096",
-        "IPY_MODEL_69da672452a94ea68b438b3be68f1a24",
-        "IPY_MODEL_38bdea7d9b64452ea8420690243965cf"
-       ],
-       "layout": "IPY_MODEL_217459b8733641fda6a21621dc147583",
-       "tabbable": null,
-       "tooltip": null
-      }
-     },
-     "f50d3b1ce11a44d8b5801bdd32a86fa8": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "FloatProgressModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "FloatProgressModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "ProgressView",
-       "bar_style": "success",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_3383a9989d6a4375bccecc436f7df6c0",
-       "max": 26.0,
-       "min": 0.0,
-       "orientation": "horizontal",
-       "style": "IPY_MODEL_a05caff4cdcb42e8813ccaf3072ea092",
-       "tabbable": null,
-       "tooltip": null,
-       "value": 26.0
-      }
-     },
-     "f5bf41cb59ed4db4b184f40d99bfbc1a": {
+     "fa2989d4907f487fb94956f67090d1bd": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -6330,7 +6294,7 @@
        "width": null
       }
      },
-     "f825f61e07764da39007e09a988d2df0": {
+     "faa2abf5d66c4dfe96ce5fec6486692e": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -6383,7 +6347,30 @@
        "width": null
       }
      },
-     "fdf5d6542de342d08a2d9a0482eda1d7": {
+     "fca8d2f8306640ba9f2e11c6d492392e": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_85c96ee49b1e46fdafef7b1f934ef9ad",
+       "placeholder": "​",
+       "style": "IPY_MODEL_30ad307a0269480eb94b844695317e07",
+       "tabbable": null,
+       "tooltip": null,
+       "value": " 50/50 [00:02&lt;00:00, 23.96it/s]"
+      }
+     },
+     "ffd00559da434d5ea3a744e89e8ab013": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLModel",
@@ -6398,12 +6385,12 @@
        "_view_name": "HTMLView",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_22fa2830a0ab4ae4bb190db8ab08f59c",
+       "layout": "IPY_MODEL_82787d08d40948da9d57fa80681d6c01",
        "placeholder": "​",
-       "style": "IPY_MODEL_bd751bbba24c4154acc75b88e0f3a87a",
+       "style": "IPY_MODEL_895aa0507aab41399395c24c2da2a903",
        "tabbable": null,
        "tooltip": null,
-       "value": " 12/12 [00:19&lt;00:00,  1.65s/it]"
+       "value": " 456k/456k [00:00&lt;00:00, 3.49MB/s]"
       }
      }
     },
diff --git a/generated/model_properties_table.html b/generated/model_properties_table.html
index 692d87b52..e3e2b71dd 100644
--- a/generated/model_properties_table.html
+++ b/generated/model_properties_table.html
@@ -1599,31 +1599,55 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>28672</p></td>
 <td class="text-left"><p>8</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>meta-llama/Llama-3.2-1B</p></td>
-<td class="text-left"><p>1.1B</p></td>
-<td class="text-right"><p>16</p></td>
+<tr class="row-odd"><td class="text-left"><p>meta-llama/Llama-3.1-70B</p></td>
+<td class="text-left"><p>78B</p></td>
+<td class="text-right"><p>80</p></td>
+<td class="text-right"><p>8192</p></td>
+<td class="text-right"><p>64</p></td>
+<td class="text-left"><p>silu</p></td>
 <td class="text-right"><p>2048</p></td>
+<td class="text-right"><p>128256</p></td>
+<td class="text-right"><p>128</p></td>
+<td class="text-right"><p>28672</p></td>
+<td class="text-left"><p>8</p></td>
+</tr>
+<tr class="row-even"><td class="text-left"><p>meta-llama/Llama-3.1-8B</p></td>
+<td class="text-left"><p>7.8B</p></td>
+<td class="text-right"><p>32</p></td>
+<td class="text-right"><p>4096</p></td>
 <td class="text-right"><p>32</p></td>
 <td class="text-left"><p>silu</p></td>
 <td class="text-right"><p>2048</p></td>
 <td class="text-right"><p>128256</p></td>
-<td class="text-right"><p>64</p></td>
-<td class="text-right"><p>8192</p></td>
+<td class="text-right"><p>128</p></td>
+<td class="text-right"><p>14336</p></td>
 <td class="text-left"><p>8</p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>meta-llama/Llama-3.2-3B</p></td>
-<td class="text-left"><p>3.2B</p></td>
-<td class="text-right"><p>28</p></td>
-<td class="text-right"><p>3072</p></td>
-<td class="text-right"><p>24</p></td>
+<tr class="row-odd"><td class="text-left"><p>meta-llama/Llama-3.1-8B-Instruct</p></td>
+<td class="text-left"><p>7.8B</p></td>
+<td class="text-right"><p>32</p></td>
+<td class="text-right"><p>4096</p></td>
+<td class="text-right"><p>32</p></td>
 <td class="text-left"><p>silu</p></td>
 <td class="text-right"><p>2048</p></td>
 <td class="text-right"><p>128256</p></td>
 <td class="text-right"><p>128</p></td>
+<td class="text-right"><p>14336</p></td>
+<td class="text-left"><p>8</p></td>
+</tr>
+<tr class="row-even"><td class="text-left"><p>meta-llama/Llama-3.1-70B-Instruct</p></td>
+<td class="text-left"><p>78B</p></td>
+<td class="text-right"><p>80</p></td>
 <td class="text-right"><p>8192</p></td>
+<td class="text-right"><p>64</p></td>
+<td class="text-left"><p>silu</p></td>
+<td class="text-right"><p>2048</p></td>
+<td class="text-right"><p>128256</p></td>
+<td class="text-right"><p>128</p></td>
+<td class="text-right"><p>28672</p></td>
 <td class="text-left"><p>8</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>meta-llama/Llama-3.2-1B-Instruct</p></td>
+<tr class="row-odd"><td class="text-left"><p>meta-llama/Llama-3.2-1B</p></td>
 <td class="text-left"><p>1.1B</p></td>
 <td class="text-right"><p>16</p></td>
 <td class="text-right"><p>2048</p></td>
@@ -1635,7 +1659,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>8192</p></td>
 <td class="text-left"><p>8</p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>meta-llama/Llama-3.2-3B-Instruct</p></td>
+<tr class="row-even"><td class="text-left"><p>meta-llama/Llama-3.2-3B</p></td>
 <td class="text-left"><p>3.2B</p></td>
 <td class="text-right"><p>28</p></td>
 <td class="text-right"><p>3072</p></td>
@@ -1647,43 +1671,31 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>8192</p></td>
 <td class="text-left"><p>8</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>meta-llama/Llama-3.1-70B</p></td>
-<td class="text-left"><p>78B</p></td>
-<td class="text-right"><p>80</p></td>
-<td class="text-right"><p>8192</p></td>
-<td class="text-right"><p>64</p></td>
-<td class="text-left"><p>silu</p></td>
+<tr class="row-odd"><td class="text-left"><p>meta-llama/Llama-3.2-1B-Instruct</p></td>
+<td class="text-left"><p>1.1B</p></td>
+<td class="text-right"><p>16</p></td>
 <td class="text-right"><p>2048</p></td>
-<td class="text-right"><p>128256</p></td>
-<td class="text-right"><p>128</p></td>
-<td class="text-right"><p>28672</p></td>
-<td class="text-left"><p>8</p></td>
-</tr>
-<tr class="row-even"><td class="text-left"><p>meta-llama/Llama-3.1-8B</p></td>
-<td class="text-left"><p>7.8B</p></td>
-<td class="text-right"><p>32</p></td>
-<td class="text-right"><p>4096</p></td>
 <td class="text-right"><p>32</p></td>
 <td class="text-left"><p>silu</p></td>
 <td class="text-right"><p>2048</p></td>
 <td class="text-right"><p>128256</p></td>
-<td class="text-right"><p>128</p></td>
-<td class="text-right"><p>14336</p></td>
+<td class="text-right"><p>64</p></td>
+<td class="text-right"><p>8192</p></td>
 <td class="text-left"><p>8</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>meta-llama/Llama-3.1-8B-Instruct</p></td>
-<td class="text-left"><p>7.8B</p></td>
-<td class="text-right"><p>32</p></td>
-<td class="text-right"><p>4096</p></td>
-<td class="text-right"><p>32</p></td>
+<tr class="row-even"><td class="text-left"><p>meta-llama/Llama-3.2-3B-Instruct</p></td>
+<td class="text-left"><p>3.2B</p></td>
+<td class="text-right"><p>28</p></td>
+<td class="text-right"><p>3072</p></td>
+<td class="text-right"><p>24</p></td>
 <td class="text-left"><p>silu</p></td>
 <td class="text-right"><p>2048</p></td>
 <td class="text-right"><p>128256</p></td>
 <td class="text-right"><p>128</p></td>
-<td class="text-right"><p>14336</p></td>
+<td class="text-right"><p>8192</p></td>
 <td class="text-left"><p>8</p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>meta-llama/Llama-3.1-70B-Instruct</p></td>
+<tr class="row-odd"><td class="text-left"><p>meta-llama/Llama-3.3-70B-Instruct</p></td>
 <td class="text-left"><p>78B</p></td>
 <td class="text-right"><p>80</p></td>
 <td class="text-right"><p>8192</p></td>
@@ -1695,7 +1707,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>28672</p></td>
 <td class="text-left"><p>8</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>othello-gpt</p></td>
+<tr class="row-even"><td class="text-left"><p>othello-gpt</p></td>
 <td class="text-left"><p>25M</p></td>
 <td class="text-right"><p>8</p></td>
 <td class="text-right"><p>512</p></td>
@@ -1707,7 +1719,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>2048</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>bert-base-cased</p></td>
+<tr class="row-odd"><td class="text-left"><p>bert-base-cased</p></td>
 <td class="text-left"><p>85M</p></td>
 <td class="text-right"><p>12</p></td>
 <td class="text-right"><p>768</p></td>
@@ -1719,7 +1731,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>3072</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>tiny-stories-1M</p></td>
+<tr class="row-even"><td class="text-left"><p>tiny-stories-1M</p></td>
 <td class="text-left"><p>393K</p></td>
 <td class="text-right"><p>8</p></td>
 <td class="text-right"><p>64</p></td>
@@ -1731,7 +1743,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>256</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>tiny-stories-3M</p></td>
+<tr class="row-odd"><td class="text-left"><p>tiny-stories-3M</p></td>
 <td class="text-left"><p>1.6M</p></td>
 <td class="text-right"><p>8</p></td>
 <td class="text-right"><p>128</p></td>
@@ -1743,7 +1755,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>512</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>tiny-stories-8M</p></td>
+<tr class="row-even"><td class="text-left"><p>tiny-stories-8M</p></td>
 <td class="text-left"><p>6.3M</p></td>
 <td class="text-right"><p>8</p></td>
 <td class="text-right"><p>256</p></td>
@@ -1755,7 +1767,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>1024</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>tiny-stories-28M</p></td>
+<tr class="row-odd"><td class="text-left"><p>tiny-stories-28M</p></td>
 <td class="text-left"><p>25M</p></td>
 <td class="text-right"><p>8</p></td>
 <td class="text-right"><p>512</p></td>
@@ -1767,7 +1779,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>2048</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>tiny-stories-33M</p></td>
+<tr class="row-even"><td class="text-left"><p>tiny-stories-33M</p></td>
 <td class="text-left"><p>28M</p></td>
 <td class="text-right"><p>4</p></td>
 <td class="text-right"><p>768</p></td>
@@ -1779,7 +1791,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>3072</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>tiny-stories-instruct-1M</p></td>
+<tr class="row-odd"><td class="text-left"><p>tiny-stories-instruct-1M</p></td>
 <td class="text-left"><p>393K</p></td>
 <td class="text-right"><p>8</p></td>
 <td class="text-right"><p>64</p></td>
@@ -1791,7 +1803,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>256</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>tiny-stories-instruct-3M</p></td>
+<tr class="row-even"><td class="text-left"><p>tiny-stories-instruct-3M</p></td>
 <td class="text-left"><p>1.6M</p></td>
 <td class="text-right"><p>8</p></td>
 <td class="text-right"><p>128</p></td>
@@ -1803,7 +1815,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>512</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>tiny-stories-instruct-8M</p></td>
+<tr class="row-odd"><td class="text-left"><p>tiny-stories-instruct-8M</p></td>
 <td class="text-left"><p>6.3M</p></td>
 <td class="text-right"><p>8</p></td>
 <td class="text-right"><p>256</p></td>
@@ -1815,7 +1827,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>1024</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>tiny-stories-instruct-28M</p></td>
+<tr class="row-even"><td class="text-left"><p>tiny-stories-instruct-28M</p></td>
 <td class="text-left"><p>25M</p></td>
 <td class="text-right"><p>8</p></td>
 <td class="text-right"><p>512</p></td>
@@ -1827,7 +1839,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>2048</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>tiny-stories-instruct-33M</p></td>
+<tr class="row-odd"><td class="text-left"><p>tiny-stories-instruct-33M</p></td>
 <td class="text-left"><p>28M</p></td>
 <td class="text-right"><p>4</p></td>
 <td class="text-right"><p>768</p></td>
@@ -1839,7 +1851,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>3072</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>tiny-stories-1L-21M</p></td>
+<tr class="row-even"><td class="text-left"><p>tiny-stories-1L-21M</p></td>
 <td class="text-left"><p>13M</p></td>
 <td class="text-right"><p>1</p></td>
 <td class="text-right"><p>1024</p></td>
@@ -1851,7 +1863,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>4096</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>tiny-stories-2L-33M</p></td>
+<tr class="row-odd"><td class="text-left"><p>tiny-stories-2L-33M</p></td>
 <td class="text-left"><p>25M</p></td>
 <td class="text-right"><p>2</p></td>
 <td class="text-right"><p>1024</p></td>
@@ -1863,7 +1875,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>4096</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>tiny-stories-instruct-1L-21M</p></td>
+<tr class="row-even"><td class="text-left"><p>tiny-stories-instruct-1L-21M</p></td>
 <td class="text-left"><p>13M</p></td>
 <td class="text-right"><p>1</p></td>
 <td class="text-right"><p>1024</p></td>
@@ -1875,7 +1887,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>4096</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>tiny-stories-instruct-2L-33M</p></td>
+<tr class="row-odd"><td class="text-left"><p>tiny-stories-instruct-2L-33M</p></td>
 <td class="text-left"><p>25M</p></td>
 <td class="text-right"><p>2</p></td>
 <td class="text-right"><p>1024</p></td>
@@ -1887,7 +1899,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>4096</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>stablelm-base-alpha-3b</p></td>
+<tr class="row-even"><td class="text-left"><p>stablelm-base-alpha-3b</p></td>
 <td class="text-left"><p>3.2B</p></td>
 <td class="text-right"><p>16</p></td>
 <td class="text-right"><p>4096</p></td>
@@ -1899,7 +1911,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>16384</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>stablelm-base-alpha-7b</p></td>
+<tr class="row-odd"><td class="text-left"><p>stablelm-base-alpha-7b</p></td>
 <td class="text-left"><p>7.2B</p></td>
 <td class="text-right"><p>16</p></td>
 <td class="text-right"><p>6144</p></td>
@@ -1911,7 +1923,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>24576</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>stablelm-tuned-alpha-3b</p></td>
+<tr class="row-even"><td class="text-left"><p>stablelm-tuned-alpha-3b</p></td>
 <td class="text-left"><p>3.2B</p></td>
 <td class="text-right"><p>16</p></td>
 <td class="text-right"><p>4096</p></td>
@@ -1923,7 +1935,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>16384</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>stablelm-tuned-alpha-7b</p></td>
+<tr class="row-odd"><td class="text-left"><p>stablelm-tuned-alpha-7b</p></td>
 <td class="text-left"><p>7.2B</p></td>
 <td class="text-right"><p>16</p></td>
 <td class="text-right"><p>6144</p></td>
@@ -1935,7 +1947,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>24576</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>mistral-7b</p></td>
+<tr class="row-even"><td class="text-left"><p>mistral-7b</p></td>
 <td class="text-left"><p>7.8B</p></td>
 <td class="text-right"><p>32</p></td>
 <td class="text-right"><p>4096</p></td>
@@ -1947,7 +1959,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>14336</p></td>
 <td class="text-left"><p>8</p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>mistral-7b-instruct</p></td>
+<tr class="row-odd"><td class="text-left"><p>mistral-7b-instruct</p></td>
 <td class="text-left"><p>7.8B</p></td>
 <td class="text-right"><p>32</p></td>
 <td class="text-right"><p>4096</p></td>
@@ -1959,7 +1971,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>14336</p></td>
 <td class="text-left"><p>8</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>mistral-nemo-base-2407</p></td>
+<tr class="row-even"><td class="text-left"><p>mistral-nemo-base-2407</p></td>
 <td class="text-left"><p>12B</p></td>
 <td class="text-right"><p>40</p></td>
 <td class="text-right"><p>5120</p></td>
@@ -1971,7 +1983,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>14336</p></td>
 <td class="text-left"><p>8</p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>mixtral</p></td>
+<tr class="row-odd"><td class="text-left"><p>mixtral</p></td>
 <td class="text-left"><p>47B</p></td>
 <td class="text-right"><p>32</p></td>
 <td class="text-right"><p>4096</p></td>
@@ -1983,7 +1995,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>14336</p></td>
 <td class="text-left"><p>8</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>mixtral-instruct</p></td>
+<tr class="row-even"><td class="text-left"><p>mixtral-instruct</p></td>
 <td class="text-left"><p>47B</p></td>
 <td class="text-right"><p>32</p></td>
 <td class="text-right"><p>4096</p></td>
@@ -1995,7 +2007,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>14336</p></td>
 <td class="text-left"><p>8</p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>bloom-560m</p></td>
+<tr class="row-odd"><td class="text-left"><p>bloom-560m</p></td>
 <td class="text-left"><p>302M</p></td>
 <td class="text-right"><p>24</p></td>
 <td class="text-right"><p>1024</p></td>
@@ -2007,7 +2019,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>4096</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>bloom-1b1</p></td>
+<tr class="row-even"><td class="text-left"><p>bloom-1b1</p></td>
 <td class="text-left"><p>679M</p></td>
 <td class="text-right"><p>24</p></td>
 <td class="text-right"><p>1536</p></td>
@@ -2019,7 +2031,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>6144</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>bloom-1b7</p></td>
+<tr class="row-odd"><td class="text-left"><p>bloom-1b7</p></td>
 <td class="text-left"><p>1.2B</p></td>
 <td class="text-right"><p>24</p></td>
 <td class="text-right"><p>2048</p></td>
@@ -2031,7 +2043,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>8192</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>bloom-3b</p></td>
+<tr class="row-even"><td class="text-left"><p>bloom-3b</p></td>
 <td class="text-left"><p>2.4B</p></td>
 <td class="text-right"><p>30</p></td>
 <td class="text-right"><p>2560</p></td>
@@ -2043,7 +2055,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>10240</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>bloom-7b1</p></td>
+<tr class="row-odd"><td class="text-left"><p>bloom-7b1</p></td>
 <td class="text-left"><p>6.0B</p></td>
 <td class="text-right"><p>30</p></td>
 <td class="text-right"><p>4096</p></td>
@@ -2055,7 +2067,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>16384</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>santacoder</p></td>
+<tr class="row-even"><td class="text-left"><p>santacoder</p></td>
 <td class="text-left"><p>1.2B</p></td>
 <td class="text-right"><p>24</p></td>
 <td class="text-right"><p>2048</p></td>
@@ -2067,7 +2079,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>8192</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>qwen-1.8b</p></td>
+<tr class="row-odd"><td class="text-left"><p>qwen-1.8b</p></td>
 <td class="text-left"><p>1.2B</p></td>
 <td class="text-right"><p>24</p></td>
 <td class="text-right"><p>2048</p></td>
@@ -2079,7 +2091,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>5504</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>qwen-7b</p></td>
+<tr class="row-even"><td class="text-left"><p>qwen-7b</p></td>
 <td class="text-left"><p>6.5B</p></td>
 <td class="text-right"><p>32</p></td>
 <td class="text-right"><p>4096</p></td>
@@ -2091,7 +2103,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>11008</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>qwen-14b</p></td>
+<tr class="row-odd"><td class="text-left"><p>qwen-14b</p></td>
 <td class="text-left"><p>13B</p></td>
 <td class="text-right"><p>40</p></td>
 <td class="text-right"><p>5120</p></td>
@@ -2103,7 +2115,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>13696</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>qwen-1.8b-chat</p></td>
+<tr class="row-even"><td class="text-left"><p>qwen-1.8b-chat</p></td>
 <td class="text-left"><p>1.2B</p></td>
 <td class="text-right"><p>24</p></td>
 <td class="text-right"><p>2048</p></td>
@@ -2115,7 +2127,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>5504</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>qwen-7b-chat</p></td>
+<tr class="row-odd"><td class="text-left"><p>qwen-7b-chat</p></td>
 <td class="text-left"><p>6.5B</p></td>
 <td class="text-right"><p>32</p></td>
 <td class="text-right"><p>4096</p></td>
@@ -2127,7 +2139,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>11008</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>qwen-14b-chat</p></td>
+<tr class="row-even"><td class="text-left"><p>qwen-14b-chat</p></td>
 <td class="text-left"><p>13B</p></td>
 <td class="text-right"><p>40</p></td>
 <td class="text-right"><p>5120</p></td>
@@ -2139,7 +2151,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>13696</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>qwen1.5-0.5b</p></td>
+<tr class="row-odd"><td class="text-left"><p>qwen1.5-0.5b</p></td>
 <td class="text-left"><p>308M</p></td>
 <td class="text-right"><p>24</p></td>
 <td class="text-right"><p>1024</p></td>
@@ -2151,7 +2163,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>2816</p></td>
 <td class="text-left"><p>16</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>qwen1.5-0.5b-chat</p></td>
+<tr class="row-even"><td class="text-left"><p>qwen1.5-0.5b-chat</p></td>
 <td class="text-left"><p>308M</p></td>
 <td class="text-right"><p>24</p></td>
 <td class="text-right"><p>1024</p></td>
@@ -2163,7 +2175,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>2816</p></td>
 <td class="text-left"><p>16</p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>qwen1.5-1.8b</p></td>
+<tr class="row-odd"><td class="text-left"><p>qwen1.5-1.8b</p></td>
 <td class="text-left"><p>1.2B</p></td>
 <td class="text-right"><p>24</p></td>
 <td class="text-right"><p>2048</p></td>
@@ -2175,7 +2187,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>5504</p></td>
 <td class="text-left"><p>16</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>qwen1.5-1.8b-chat</p></td>
+<tr class="row-even"><td class="text-left"><p>qwen1.5-1.8b-chat</p></td>
 <td class="text-left"><p>1.2B</p></td>
 <td class="text-right"><p>24</p></td>
 <td class="text-right"><p>2048</p></td>
@@ -2187,7 +2199,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>5504</p></td>
 <td class="text-left"><p>16</p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>qwen1.5-4b</p></td>
+<tr class="row-odd"><td class="text-left"><p>qwen1.5-4b</p></td>
 <td class="text-left"><p>3.2B</p></td>
 <td class="text-right"><p>40</p></td>
 <td class="text-right"><p>2560</p></td>
@@ -2199,7 +2211,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>6912</p></td>
 <td class="text-left"><p>20</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>qwen1.5-4b-chat</p></td>
+<tr class="row-even"><td class="text-left"><p>qwen1.5-4b-chat</p></td>
 <td class="text-left"><p>3.2B</p></td>
 <td class="text-right"><p>40</p></td>
 <td class="text-right"><p>2560</p></td>
@@ -2211,7 +2223,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>6912</p></td>
 <td class="text-left"><p>20</p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>qwen1.5-7b</p></td>
+<tr class="row-odd"><td class="text-left"><p>qwen1.5-7b</p></td>
 <td class="text-left"><p>6.5B</p></td>
 <td class="text-right"><p>32</p></td>
 <td class="text-right"><p>4096</p></td>
@@ -2223,7 +2235,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>11008</p></td>
 <td class="text-left"><p>32</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>qwen1.5-7b-chat</p></td>
+<tr class="row-even"><td class="text-left"><p>qwen1.5-7b-chat</p></td>
 <td class="text-left"><p>6.5B</p></td>
 <td class="text-right"><p>32</p></td>
 <td class="text-right"><p>4096</p></td>
@@ -2235,7 +2247,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>11008</p></td>
 <td class="text-left"><p>32</p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>qwen1.5-14b</p></td>
+<tr class="row-odd"><td class="text-left"><p>qwen1.5-14b</p></td>
 <td class="text-left"><p>13B</p></td>
 <td class="text-right"><p>40</p></td>
 <td class="text-right"><p>5120</p></td>
@@ -2247,7 +2259,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>13696</p></td>
 <td class="text-left"><p>40</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>qwen1.5-14b-chat</p></td>
+<tr class="row-even"><td class="text-left"><p>qwen1.5-14b-chat</p></td>
 <td class="text-left"><p>13B</p></td>
 <td class="text-right"><p>40</p></td>
 <td class="text-right"><p>5120</p></td>
@@ -2259,7 +2271,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>13696</p></td>
 <td class="text-left"><p>40</p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>Qwen/Qwen2-0.5B</p></td>
+<tr class="row-odd"><td class="text-left"><p>Qwen/Qwen2-0.5B</p></td>
 <td class="text-left"><p>391M</p></td>
 <td class="text-right"><p>24</p></td>
 <td class="text-right"><p>896</p></td>
@@ -2271,7 +2283,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>4864</p></td>
 <td class="text-left"><p>2</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>Qwen/Qwen2-0.5B-Instruct</p></td>
+<tr class="row-even"><td class="text-left"><p>Qwen/Qwen2-0.5B-Instruct</p></td>
 <td class="text-left"><p>391M</p></td>
 <td class="text-right"><p>24</p></td>
 <td class="text-right"><p>896</p></td>
@@ -2283,7 +2295,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>4864</p></td>
 <td class="text-left"><p>2</p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>Qwen/Qwen2-1.5B</p></td>
+<tr class="row-odd"><td class="text-left"><p>Qwen/Qwen2-1.5B</p></td>
 <td class="text-left"><p>1.4B</p></td>
 <td class="text-right"><p>28</p></td>
 <td class="text-right"><p>1536</p></td>
@@ -2295,7 +2307,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>8960</p></td>
 <td class="text-left"><p>2</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>Qwen/Qwen2-1.5B-Instruct</p></td>
+<tr class="row-even"><td class="text-left"><p>Qwen/Qwen2-1.5B-Instruct</p></td>
 <td class="text-left"><p>1.4B</p></td>
 <td class="text-right"><p>28</p></td>
 <td class="text-right"><p>1536</p></td>
@@ -2307,7 +2319,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>8960</p></td>
 <td class="text-left"><p>2</p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>Qwen/Qwen2-7B</p></td>
+<tr class="row-odd"><td class="text-left"><p>Qwen/Qwen2-7B</p></td>
 <td class="text-left"><p>7.1B</p></td>
 <td class="text-right"><p>28</p></td>
 <td class="text-right"><p>3584</p></td>
@@ -2319,7 +2331,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>18944</p></td>
 <td class="text-left"><p>4</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>Qwen/Qwen2-7B-Instruct</p></td>
+<tr class="row-even"><td class="text-left"><p>Qwen/Qwen2-7B-Instruct</p></td>
 <td class="text-left"><p>7.1B</p></td>
 <td class="text-right"><p>28</p></td>
 <td class="text-right"><p>3584</p></td>
@@ -2331,7 +2343,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>18944</p></td>
 <td class="text-left"><p>4</p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>Qwen/Qwen2.5-0.5B</p></td>
+<tr class="row-odd"><td class="text-left"><p>Qwen/Qwen2.5-0.5B</p></td>
 <td class="text-left"><p>391M</p></td>
 <td class="text-right"><p>24</p></td>
 <td class="text-right"><p>896</p></td>
@@ -2343,7 +2355,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>4864</p></td>
 <td class="text-left"><p>2</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>Qwen/Qwen2.5-0.5B-Instruct</p></td>
+<tr class="row-even"><td class="text-left"><p>Qwen/Qwen2.5-0.5B-Instruct</p></td>
 <td class="text-left"><p>391M</p></td>
 <td class="text-right"><p>24</p></td>
 <td class="text-right"><p>896</p></td>
@@ -2355,7 +2367,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>4864</p></td>
 <td class="text-left"><p>2</p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>Qwen/Qwen2.5-1.5B</p></td>
+<tr class="row-odd"><td class="text-left"><p>Qwen/Qwen2.5-1.5B</p></td>
 <td class="text-left"><p>1.4B</p></td>
 <td class="text-right"><p>28</p></td>
 <td class="text-right"><p>1536</p></td>
@@ -2367,7 +2379,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>8960</p></td>
 <td class="text-left"><p>2</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>Qwen/Qwen2.5-1.5B-Instruct</p></td>
+<tr class="row-even"><td class="text-left"><p>Qwen/Qwen2.5-1.5B-Instruct</p></td>
 <td class="text-left"><p>1.4B</p></td>
 <td class="text-right"><p>28</p></td>
 <td class="text-right"><p>1536</p></td>
@@ -2379,7 +2391,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>8960</p></td>
 <td class="text-left"><p>2</p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>Qwen/Qwen2.5-3B</p></td>
+<tr class="row-odd"><td class="text-left"><p>Qwen/Qwen2.5-3B</p></td>
 <td class="text-left"><p>3.0B</p></td>
 <td class="text-right"><p>36</p></td>
 <td class="text-right"><p>2048</p></td>
@@ -2391,7 +2403,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>11008</p></td>
 <td class="text-left"><p>2</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>Qwen/Qwen2.5-3B-Instruct</p></td>
+<tr class="row-even"><td class="text-left"><p>Qwen/Qwen2.5-3B-Instruct</p></td>
 <td class="text-left"><p>3.0B</p></td>
 <td class="text-right"><p>36</p></td>
 <td class="text-right"><p>2048</p></td>
@@ -2403,7 +2415,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>11008</p></td>
 <td class="text-left"><p>2</p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>Qwen/Qwen2.5-7B</p></td>
+<tr class="row-odd"><td class="text-left"><p>Qwen/Qwen2.5-7B</p></td>
 <td class="text-left"><p>7.1B</p></td>
 <td class="text-right"><p>28</p></td>
 <td class="text-right"><p>3584</p></td>
@@ -2415,7 +2427,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>18944</p></td>
 <td class="text-left"><p>4</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>Qwen/Qwen2.5-7B-Instruct</p></td>
+<tr class="row-even"><td class="text-left"><p>Qwen/Qwen2.5-7B-Instruct</p></td>
 <td class="text-left"><p>7.1B</p></td>
 <td class="text-right"><p>28</p></td>
 <td class="text-right"><p>3584</p></td>
@@ -2427,7 +2439,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>18944</p></td>
 <td class="text-left"><p>4</p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>Qwen/Qwen2.5-14B</p></td>
+<tr class="row-odd"><td class="text-left"><p>Qwen/Qwen2.5-14B</p></td>
 <td class="text-left"><p>15B</p></td>
 <td class="text-right"><p>48</p></td>
 <td class="text-right"><p>5120</p></td>
@@ -2439,7 +2451,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>13824</p></td>
 <td class="text-left"><p>8</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>Qwen/Qwen2.5-14B-Instruct</p></td>
+<tr class="row-even"><td class="text-left"><p>Qwen/Qwen2.5-14B-Instruct</p></td>
 <td class="text-left"><p>15B</p></td>
 <td class="text-right"><p>48</p></td>
 <td class="text-right"><p>5120</p></td>
@@ -2451,7 +2463,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>13824</p></td>
 <td class="text-left"><p>8</p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>Qwen/Qwen2.5-32B</p></td>
+<tr class="row-odd"><td class="text-left"><p>Qwen/Qwen2.5-32B</p></td>
 <td class="text-left"><p>34B</p></td>
 <td class="text-right"><p>64</p></td>
 <td class="text-right"><p>5120</p></td>
@@ -2463,7 +2475,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>27648</p></td>
 <td class="text-left"><p>8</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>Qwen/Qwen2.5-32B-Instruct</p></td>
+<tr class="row-even"><td class="text-left"><p>Qwen/Qwen2.5-32B-Instruct</p></td>
 <td class="text-left"><p>34B</p></td>
 <td class="text-right"><p>64</p></td>
 <td class="text-right"><p>5120</p></td>
@@ -2475,7 +2487,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>27648</p></td>
 <td class="text-left"><p>8</p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>Qwen/Qwen2.5-72B</p></td>
+<tr class="row-odd"><td class="text-left"><p>Qwen/Qwen2.5-72B</p></td>
 <td class="text-left"><p>80B</p></td>
 <td class="text-right"><p>80</p></td>
 <td class="text-right"><p>8192</p></td>
@@ -2487,7 +2499,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>29568</p></td>
 <td class="text-left"><p>8</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>Qwen/Qwen2.5-72B-Instruct</p></td>
+<tr class="row-even"><td class="text-left"><p>Qwen/Qwen2.5-72B-Instruct</p></td>
 <td class="text-left"><p>80B</p></td>
 <td class="text-right"><p>80</p></td>
 <td class="text-right"><p>8192</p></td>
@@ -2499,7 +2511,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>29568</p></td>
 <td class="text-left"><p>8</p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>Qwen/QwQ-32B-Preview</p></td>
+<tr class="row-odd"><td class="text-left"><p>Qwen/QwQ-32B-Preview</p></td>
 <td class="text-left"><p>34B</p></td>
 <td class="text-right"><p>64</p></td>
 <td class="text-right"><p>5120</p></td>
@@ -2511,7 +2523,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>27648</p></td>
 <td class="text-left"><p>8</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>phi-1</p></td>
+<tr class="row-even"><td class="text-left"><p>phi-1</p></td>
 <td class="text-left"><p>1.2B</p></td>
 <td class="text-right"><p>24</p></td>
 <td class="text-right"><p>2048</p></td>
@@ -2523,7 +2535,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>8192</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>phi-1_5</p></td>
+<tr class="row-odd"><td class="text-left"><p>phi-1_5</p></td>
 <td class="text-left"><p>1.2B</p></td>
 <td class="text-right"><p>24</p></td>
 <td class="text-right"><p>2048</p></td>
@@ -2535,7 +2547,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>8192</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>phi-2</p></td>
+<tr class="row-even"><td class="text-left"><p>phi-2</p></td>
 <td class="text-left"><p>2.5B</p></td>
 <td class="text-right"><p>32</p></td>
 <td class="text-right"><p>2560</p></td>
@@ -2547,7 +2559,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>10240</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>phi-3</p></td>
+<tr class="row-odd"><td class="text-left"><p>phi-3</p></td>
 <td class="text-left"><p>3.6B</p></td>
 <td class="text-right"><p>32</p></td>
 <td class="text-right"><p>3072</p></td>
@@ -2559,7 +2571,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>8192</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>gemma-2b</p></td>
+<tr class="row-even"><td class="text-left"><p>gemma-2b</p></td>
 <td class="text-left"><p>2.1B</p></td>
 <td class="text-right"><p>18</p></td>
 <td class="text-right"><p>2048</p></td>
@@ -2571,7 +2583,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>16384</p></td>
 <td class="text-left"><p>1</p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>gemma-7b</p></td>
+<tr class="row-odd"><td class="text-left"><p>gemma-7b</p></td>
 <td class="text-left"><p>7.8B</p></td>
 <td class="text-right"><p>28</p></td>
 <td class="text-right"><p>3072</p></td>
@@ -2583,7 +2595,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>24576</p></td>
 <td class="text-left"><p>16</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>gemma-2b-it</p></td>
+<tr class="row-even"><td class="text-left"><p>gemma-2b-it</p></td>
 <td class="text-left"><p>2.1B</p></td>
 <td class="text-right"><p>18</p></td>
 <td class="text-right"><p>2048</p></td>
@@ -2595,7 +2607,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>16384</p></td>
 <td class="text-left"><p>1</p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>gemma-7b-it</p></td>
+<tr class="row-odd"><td class="text-left"><p>gemma-7b-it</p></td>
 <td class="text-left"><p>7.8B</p></td>
 <td class="text-right"><p>28</p></td>
 <td class="text-right"><p>3072</p></td>
@@ -2607,7 +2619,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>24576</p></td>
 <td class="text-left"><p>16</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>gemma-2-2b</p></td>
+<tr class="row-even"><td class="text-left"><p>gemma-2-2b</p></td>
 <td class="text-left"><p>2.1B</p></td>
 <td class="text-right"><p>26</p></td>
 <td class="text-right"><p>2304</p></td>
@@ -2619,7 +2631,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>9216</p></td>
 <td class="text-left"><p>4</p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>gemma-2-2b-it</p></td>
+<tr class="row-odd"><td class="text-left"><p>gemma-2-2b-it</p></td>
 <td class="text-left"><p>2.1B</p></td>
 <td class="text-right"><p>26</p></td>
 <td class="text-right"><p>2304</p></td>
@@ -2631,7 +2643,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>9216</p></td>
 <td class="text-left"><p>4</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>gemma-2-9b</p></td>
+<tr class="row-even"><td class="text-left"><p>gemma-2-9b</p></td>
 <td class="text-left"><p>8.9B</p></td>
 <td class="text-right"><p>42</p></td>
 <td class="text-right"><p>3584</p></td>
@@ -2643,7 +2655,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>14336</p></td>
 <td class="text-left"><p>8</p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>gemma-2-9b-it</p></td>
+<tr class="row-odd"><td class="text-left"><p>gemma-2-9b-it</p></td>
 <td class="text-left"><p>8.9B</p></td>
 <td class="text-right"><p>42</p></td>
 <td class="text-right"><p>3584</p></td>
@@ -2655,7 +2667,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>14336</p></td>
 <td class="text-left"><p>8</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>gemma-2-27b</p></td>
+<tr class="row-even"><td class="text-left"><p>gemma-2-27b</p></td>
 <td class="text-left"><p>27B</p></td>
 <td class="text-right"><p>46</p></td>
 <td class="text-right"><p>4608</p></td>
@@ -2667,7 +2679,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>36864</p></td>
 <td class="text-left"><p>16</p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>gemma-2-27b-it</p></td>
+<tr class="row-odd"><td class="text-left"><p>gemma-2-27b-it</p></td>
 <td class="text-left"><p>27B</p></td>
 <td class="text-right"><p>46</p></td>
 <td class="text-right"><p>4608</p></td>
@@ -2679,7 +2691,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>36864</p></td>
 <td class="text-left"><p>16</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>yi-6b</p></td>
+<tr class="row-even"><td class="text-left"><p>yi-6b</p></td>
 <td class="text-left"><p>6.5B</p></td>
 <td class="text-right"><p>32</p></td>
 <td class="text-right"><p>4096</p></td>
@@ -2691,7 +2703,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>11008</p></td>
 <td class="text-left"><p>4</p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>yi-34b</p></td>
+<tr class="row-odd"><td class="text-left"><p>yi-34b</p></td>
 <td class="text-left"><p>39B</p></td>
 <td class="text-right"><p>60</p></td>
 <td class="text-right"><p>7168</p></td>
@@ -2703,7 +2715,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>20480</p></td>
 <td class="text-left"><p>8</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>yi-6b-chat</p></td>
+<tr class="row-even"><td class="text-left"><p>yi-6b-chat</p></td>
 <td class="text-left"><p>6.5B</p></td>
 <td class="text-right"><p>32</p></td>
 <td class="text-right"><p>4096</p></td>
@@ -2715,7 +2727,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>11008</p></td>
 <td class="text-left"><p>4</p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>yi-34b-chat</p></td>
+<tr class="row-odd"><td class="text-left"><p>yi-34b-chat</p></td>
 <td class="text-left"><p>39B</p></td>
 <td class="text-right"><p>60</p></td>
 <td class="text-right"><p>7168</p></td>
@@ -2727,7 +2739,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>20480</p></td>
 <td class="text-left"><p>8</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>t5-small</p></td>
+<tr class="row-even"><td class="text-left"><p>t5-small</p></td>
 <td class="text-left"><p>19M</p></td>
 <td class="text-right"><p>6</p></td>
 <td class="text-right"><p>512</p></td>
@@ -2739,7 +2751,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>2048</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>t5-base</p></td>
+<tr class="row-odd"><td class="text-left"><p>t5-base</p></td>
 <td class="text-left"><p>85M</p></td>
 <td class="text-right"><p>12</p></td>
 <td class="text-right"><p>768</p></td>
@@ -2751,7 +2763,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>3072</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>t5-large</p></td>
+<tr class="row-even"><td class="text-left"><p>t5-large</p></td>
 <td class="text-left"><p>302M</p></td>
 <td class="text-right"><p>24</p></td>
 <td class="text-right"><p>1024</p></td>
@@ -2763,7 +2775,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>4096</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>mGPT</p></td>
+<tr class="row-odd"><td class="text-left"><p>mGPT</p></td>
 <td class="text-left"><p>1.2B</p></td>
 <td class="text-right"><p>24</p></td>
 <td class="text-right"><p>2048</p></td>
diff --git a/searchindex.js b/searchindex.js
index 18feeca65..6290e96e2 100644
--- a/searchindex.js
+++ b/searchindex.js
@@ -1 +1 @@
-Search.setIndex({"docnames": ["content/citation", "content/contributing", "content/gallery", "content/getting_started", "content/getting_started_mech_interp", "content/news/release-2.0", "content/special_cases", "content/tutorials", "generated/code/modules", "generated/code/transformer_lens", "generated/code/transformer_lens.ActivationCache", "generated/code/transformer_lens.FactoredMatrix", "generated/code/transformer_lens.HookedEncoder", "generated/code/transformer_lens.HookedEncoderDecoder", "generated/code/transformer_lens.HookedTransformer", "generated/code/transformer_lens.HookedTransformerConfig", "generated/code/transformer_lens.SVDInterpreter", "generated/code/transformer_lens.components", "generated/code/transformer_lens.components.abstract_attention", "generated/code/transformer_lens.components.attention", "generated/code/transformer_lens.components.bert_block", "generated/code/transformer_lens.components.bert_embed", "generated/code/transformer_lens.components.bert_mlm_head", "generated/code/transformer_lens.components.embed", "generated/code/transformer_lens.components.grouped_query_attention", "generated/code/transformer_lens.components.layer_norm", "generated/code/transformer_lens.components.layer_norm_pre", "generated/code/transformer_lens.components.pos_embed", "generated/code/transformer_lens.components.rms_norm", "generated/code/transformer_lens.components.rms_norm_pre", "generated/code/transformer_lens.components.t5_attention", "generated/code/transformer_lens.components.t5_block", "generated/code/transformer_lens.components.token_typed_embed", "generated/code/transformer_lens.components.transformer_block", "generated/code/transformer_lens.components.unembed", "generated/code/transformer_lens.evals", "generated/code/transformer_lens.head_detector", "generated/code/transformer_lens.hook_points", "generated/code/transformer_lens.loading_from_pretrained", "generated/code/transformer_lens.past_key_value_caching", "generated/code/transformer_lens.patching", "generated/code/transformer_lens.pretrained", "generated/code/transformer_lens.pretrained.weight_conversions", "generated/code/transformer_lens.pretrained.weight_conversions.bert", "generated/code/transformer_lens.pretrained.weight_conversions.bloom", "generated/code/transformer_lens.pretrained.weight_conversions.coder", "generated/code/transformer_lens.pretrained.weight_conversions.gemma", "generated/code/transformer_lens.pretrained.weight_conversions.gpt2", "generated/code/transformer_lens.pretrained.weight_conversions.gptj", "generated/code/transformer_lens.pretrained.weight_conversions.llama", "generated/code/transformer_lens.pretrained.weight_conversions.mingpt", "generated/code/transformer_lens.pretrained.weight_conversions.mistral", "generated/code/transformer_lens.pretrained.weight_conversions.mixtral", "generated/code/transformer_lens.pretrained.weight_conversions.nanogpt", "generated/code/transformer_lens.pretrained.weight_conversions.neel_solu_old", "generated/code/transformer_lens.pretrained.weight_conversions.neo", "generated/code/transformer_lens.pretrained.weight_conversions.neox", "generated/code/transformer_lens.pretrained.weight_conversions.opt", "generated/code/transformer_lens.pretrained.weight_conversions.phi", "generated/code/transformer_lens.pretrained.weight_conversions.phi3", "generated/code/transformer_lens.pretrained.weight_conversions.qwen", "generated/code/transformer_lens.pretrained.weight_conversions.qwen2", "generated/code/transformer_lens.pretrained.weight_conversions.t5", "generated/code/transformer_lens.train", "generated/code/transformer_lens.utilities", "generated/code/transformer_lens.utilities.activation_functions", "generated/code/transformer_lens.utilities.addmm", "generated/code/transformer_lens.utilities.attention", "generated/code/transformer_lens.utilities.devices", "generated/code/transformer_lens.utils", "generated/demos/Exploratory_Analysis_Demo", "generated/demos/Main_Demo", "generated/model_properties_table", "index"], "filenames": ["content/citation.md", "content/contributing.md", "content/gallery.md", "content/getting_started.md", "content/getting_started_mech_interp.md", "content/news/release-2.0.md", "content/special_cases.md", "content/tutorials.md", "generated/code/modules.rst", "generated/code/transformer_lens.rst", "generated/code/transformer_lens.ActivationCache.rst", "generated/code/transformer_lens.FactoredMatrix.rst", "generated/code/transformer_lens.HookedEncoder.rst", "generated/code/transformer_lens.HookedEncoderDecoder.rst", "generated/code/transformer_lens.HookedTransformer.rst", "generated/code/transformer_lens.HookedTransformerConfig.rst", "generated/code/transformer_lens.SVDInterpreter.rst", "generated/code/transformer_lens.components.rst", "generated/code/transformer_lens.components.abstract_attention.rst", "generated/code/transformer_lens.components.attention.rst", "generated/code/transformer_lens.components.bert_block.rst", "generated/code/transformer_lens.components.bert_embed.rst", "generated/code/transformer_lens.components.bert_mlm_head.rst", "generated/code/transformer_lens.components.embed.rst", "generated/code/transformer_lens.components.grouped_query_attention.rst", "generated/code/transformer_lens.components.layer_norm.rst", "generated/code/transformer_lens.components.layer_norm_pre.rst", "generated/code/transformer_lens.components.pos_embed.rst", "generated/code/transformer_lens.components.rms_norm.rst", "generated/code/transformer_lens.components.rms_norm_pre.rst", "generated/code/transformer_lens.components.t5_attention.rst", "generated/code/transformer_lens.components.t5_block.rst", "generated/code/transformer_lens.components.token_typed_embed.rst", "generated/code/transformer_lens.components.transformer_block.rst", "generated/code/transformer_lens.components.unembed.rst", "generated/code/transformer_lens.evals.rst", "generated/code/transformer_lens.head_detector.rst", "generated/code/transformer_lens.hook_points.rst", "generated/code/transformer_lens.loading_from_pretrained.rst", "generated/code/transformer_lens.past_key_value_caching.rst", "generated/code/transformer_lens.patching.rst", "generated/code/transformer_lens.pretrained.rst", "generated/code/transformer_lens.pretrained.weight_conversions.rst", "generated/code/transformer_lens.pretrained.weight_conversions.bert.rst", "generated/code/transformer_lens.pretrained.weight_conversions.bloom.rst", "generated/code/transformer_lens.pretrained.weight_conversions.coder.rst", "generated/code/transformer_lens.pretrained.weight_conversions.gemma.rst", "generated/code/transformer_lens.pretrained.weight_conversions.gpt2.rst", "generated/code/transformer_lens.pretrained.weight_conversions.gptj.rst", "generated/code/transformer_lens.pretrained.weight_conversions.llama.rst", "generated/code/transformer_lens.pretrained.weight_conversions.mingpt.rst", "generated/code/transformer_lens.pretrained.weight_conversions.mistral.rst", "generated/code/transformer_lens.pretrained.weight_conversions.mixtral.rst", "generated/code/transformer_lens.pretrained.weight_conversions.nanogpt.rst", "generated/code/transformer_lens.pretrained.weight_conversions.neel_solu_old.rst", "generated/code/transformer_lens.pretrained.weight_conversions.neo.rst", "generated/code/transformer_lens.pretrained.weight_conversions.neox.rst", "generated/code/transformer_lens.pretrained.weight_conversions.opt.rst", "generated/code/transformer_lens.pretrained.weight_conversions.phi.rst", "generated/code/transformer_lens.pretrained.weight_conversions.phi3.rst", "generated/code/transformer_lens.pretrained.weight_conversions.qwen.rst", "generated/code/transformer_lens.pretrained.weight_conversions.qwen2.rst", "generated/code/transformer_lens.pretrained.weight_conversions.t5.rst", "generated/code/transformer_lens.train.rst", "generated/code/transformer_lens.utilities.rst", "generated/code/transformer_lens.utilities.activation_functions.rst", "generated/code/transformer_lens.utilities.addmm.rst", "generated/code/transformer_lens.utilities.attention.rst", "generated/code/transformer_lens.utilities.devices.rst", "generated/code/transformer_lens.utils.rst", "generated/demos/Exploratory_Analysis_Demo.ipynb", "generated/demos/Main_Demo.ipynb", "generated/model_properties_table.md", "index.md"], "titles": ["Citation", "Contributing", "Gallery", "Getting Started", "Getting Started in Mechanistic Interpretability", "TransformerLens 2.0", "Special Cases", "Tutorials", "Transformer Lens API", "transformer_lens", "transformer_lens.ActivationCache", "transformer_lens.FactoredMatrix", "transformer_lens.HookedEncoder", "transformer_lens.HookedEncoderDecoder", "transformer_lens.HookedTransformer", "transformer_lens.HookedTransformerConfig", "transformer_lens.SVDInterpreter", "transformer_lens.components", "transformer_lens.components.abstract_attention", "transformer_lens.components.attention", "transformer_lens.components.bert_block", "transformer_lens.components.bert_embed", "transformer_lens.components.bert_mlm_head", "transformer_lens.components.embed", "transformer_lens.components.grouped_query_attention", "transformer_lens.components.layer_norm", "transformer_lens.components.layer_norm_pre", "transformer_lens.components.pos_embed", "transformer_lens.components.rms_norm", "transformer_lens.components.rms_norm_pre", "transformer_lens.components.t5_attention", "transformer_lens.components.t5_block", "transformer_lens.components.token_typed_embed", "transformer_lens.components.transformer_block", "transformer_lens.components.unembed", "transformer_lens.evals", "transformer_lens.head_detector", "transformer_lens.hook_points", "transformer_lens.loading_from_pretrained", "transformer_lens.past_key_value_caching", "transformer_lens.patching", "transformer_lens.pretrained", "transformer_lens.pretrained.weight_conversions", "transformer_lens.pretrained.weight_conversions.bert", "transformer_lens.pretrained.weight_conversions.bloom", "transformer_lens.pretrained.weight_conversions.coder", "transformer_lens.pretrained.weight_conversions.gemma", "transformer_lens.pretrained.weight_conversions.gpt2", "transformer_lens.pretrained.weight_conversions.gptj", "transformer_lens.pretrained.weight_conversions.llama", "transformer_lens.pretrained.weight_conversions.mingpt", "transformer_lens.pretrained.weight_conversions.mistral", "transformer_lens.pretrained.weight_conversions.mixtral", "transformer_lens.pretrained.weight_conversions.nanogpt", "transformer_lens.pretrained.weight_conversions.neel_solu_old", "transformer_lens.pretrained.weight_conversions.neo", "transformer_lens.pretrained.weight_conversions.neox", "transformer_lens.pretrained.weight_conversions.opt", "transformer_lens.pretrained.weight_conversions.phi", "transformer_lens.pretrained.weight_conversions.phi3", "transformer_lens.pretrained.weight_conversions.qwen", "transformer_lens.pretrained.weight_conversions.qwen2", "transformer_lens.pretrained.weight_conversions.t5", "transformer_lens.train", "transformer_lens.utilities", "transformer_lens.utilities.activation_functions", "transformer_lens.utilities.addmm", "transformer_lens.utilities.attention", "transformer_lens.utilities.devices", "transformer_lens.utils", "Exploratory Analysis Demo", "Transformer Lens Main Demo Notebook", "Model Properties Table", "TransformerLens"], "terms": {"pleas": [0, 1, 3, 4, 5, 71], "cite": 0, "thi": [0, 1, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 32, 33, 34, 35, 36, 37, 38, 39, 40, 63, 67, 68, 69, 71, 73], "librari": [0, 2, 3, 4, 5, 7, 10, 35, 69, 70], "misc": 0, "nanda2022transformerlen": 0, "titl": [0, 1, 70, 71], "transformerlen": [0, 2, 3, 4, 7, 10, 14, 18, 38, 69, 70, 71], "author": [0, 70], "neel": [0, 2, 4, 7, 14, 16, 71], "nanda": [0, 2, 4, 14, 71], "joseph": [0, 5], "bloom": [0, 5, 14, 18, 38, 42, 72], "year": 0, "2022": [0, 69], "howpublish": 0, "url": [0, 3], "http": [0, 1, 3, 7, 10, 14, 15, 18, 24, 32, 35, 36, 40, 66, 69, 70, 71], "github": [0, 1, 3, 7, 14, 66], "com": [0, 3, 7, 10, 14, 18, 66, 70, 71], "transformerlensorg": [0, 3, 7], "For": [1, 5, 10, 12, 13, 14, 18, 32, 36, 69, 70], "one": [1, 3, 4, 5, 10, 12, 13, 14, 15, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 36, 37, 38, 39, 40, 69, 70, 71, 73], "click": [1, 71], "your": [1, 3, 5, 7, 14, 15, 36, 37, 70, 71], "develop": [1, 5, 7, 70, 71], "environ": [1, 3, 73], "project": [1, 5, 7, 10, 18, 24, 63, 70], "includ": [1, 4, 5, 7, 10, 12, 13, 14, 15, 35, 36, 37, 70], "It": [1, 3, 5, 7, 10, 12, 13, 14, 15, 18, 35, 37, 40, 69, 70, 71, 73], "can": [1, 2, 3, 5, 6, 7, 10, 11, 12, 13, 14, 15, 16, 18, 19, 24, 35, 36, 37, 38, 39, 40, 69, 70, 71, 73], "us": [1, 2, 3, 5, 6, 7, 10, 12, 13, 14, 15, 16, 18, 19, 24, 26, 31, 33, 35, 36, 37, 38, 39, 40, 63, 67, 68, 69, 70, 71, 73], "local": [1, 14, 15, 18, 19, 24, 38, 69, 71], "v": [1, 10, 12, 13, 14, 15, 18, 24, 40, 70, 71], "code": [1, 4, 5, 10, 14, 15, 18, 35, 36, 37, 38, 69, 70, 71], "codespac": 1, "poetri": 1, "packag": 1, "manag": [1, 5, 10, 14, 37, 69], "instal": [1, 5, 70, 71], "follow": [1, 3, 5, 10, 14, 69, 71, 73], "also": [1, 5, 7, 10, 12, 13, 14, 15, 16, 31, 36, 37, 38, 68, 69, 70, 71], "virtual": 1, "config": [1, 14, 15, 18, 19, 24, 38, 40, 63], "virtualenv": 1, "true": [1, 10, 12, 13, 14, 15, 30, 31, 35, 36, 37, 38, 40, 68, 69, 70, 71], "dev": 1, "doc": [1, 5, 8, 10, 14, 71], "jupyt": 1, "If": [1, 3, 5, 8, 10, 12, 13, 14, 15, 25, 28, 32, 36, 37, 38, 40, 68, 69, 70, 71], "ad": [1, 7, 14, 15, 18, 37, 70, 71], "featur": [1, 3, 5, 7, 12, 13, 16, 18, 40, 69, 70, 73], "add": [1, 5, 14, 15, 18, 37, 39, 66, 69, 70, 71, 73], "unit": [1, 5], "you": [1, 3, 4, 5, 7, 10, 12, 13, 14, 15, 16, 30, 35, 36, 37, 38, 69, 70, 71, 73], "need": [1, 3, 5, 10, 14, 15, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 69, 70, 71, 73], "model": [1, 2, 3, 6, 7, 10, 12, 13, 14, 15, 16, 18, 19, 21, 24, 35, 36, 37, 38, 40, 45, 63, 68, 69, 70], "ones": [1, 12, 13, 14, 31, 36, 70], "ar": [1, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15, 18, 20, 24, 35, 36, 37, 38, 39, 40, 69, 71, 73], "cach": [1, 10, 14, 15, 31, 33, 36, 37, 39, 40, 69, 70, 73], "action": [1, 2, 70], "so": [1, 5, 7, 10, 11, 12, 13, 14, 15, 16, 35, 37, 38, 39, 40, 69, 70, 71], "quickli": [1, 5, 7, 73], "cd": [1, 70, 71], "These": [1, 70, 71], "gpt2": [1, 14, 15, 16, 18, 19, 24, 35, 38, 42, 70, 71, 72], "attn": [1, 10, 12, 13, 14, 15, 18, 30, 38, 40, 69, 70, 71, 72], "onli": [1, 2, 5, 10, 11, 12, 13, 14, 15, 18, 19, 24, 26, 31, 33, 36, 37, 38, 69, 70, 71, 72], "1l": [1, 38, 70, 71, 72], "2l": [1, 14, 38, 71, 72], "3l": [1, 38, 71, 72], "4l": [1, 38, 71, 72], "tini": [1, 10, 14, 38, 69, 70, 71, 72], "stori": [1, 10, 14, 38, 40, 69, 70, 72], "1m": [1, 10, 14, 38, 69, 72], "note": [1, 3, 5, 10, 11, 12, 13, 14, 15, 18, 24, 35, 37, 38, 66, 69, 70, 71], "i": [1, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15, 16, 18, 22, 24, 26, 31, 32, 33, 35, 36, 37, 38, 39, 40, 63, 67, 68, 69, 70, 73], "quit": [1, 5], "slow": [1, 71], "we": [1, 2, 5, 8, 10, 14, 15, 36, 39, 40, 69, 70, 71, 73], "have": [1, 3, 5, 10, 12, 13, 14, 18, 19, 24, 36, 40, 69, 70, 71, 73], "cpu": [1, 10, 12, 13, 14, 15, 18, 38, 70, 71], "smaller": [1, 5, 71], "like": [1, 3, 4, 5, 7, 12, 13, 14, 15, 21, 30, 35, 36, 40, 69, 70, 71, 73], "prefer": 1, "possibl": [1, 5, 12, 13, 14, 36, 40, 69, 70, 71, 73], "via": [1, 2, 3, 4, 5, 12, 13, 14, 40, 70], "make": [1, 3, 5, 7, 11, 12, 13, 14, 36, 37, 70, 71, 73], "accept": [1, 3, 5, 12, 13, 14, 37, 70], "notebook": [1, 3, 7, 70, 73], "all": [1, 4, 5, 10, 12, 13, 14, 15, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 32, 33, 34, 35, 36, 37, 40, 65, 69, 70, 73], "suit": 1, "mention": [1, 5, 71], "pycln": 1, "isort": 1, "black": [1, 71], "pull": [1, 5], "request": [1, 5], "check": [1, 3, 5, 7, 14, 16, 18, 35, 36, 37, 69, 70, 71], "file": [1, 5, 69], "line": [1, 5, 70, 71], "length": [1, 10, 12, 13, 14, 15, 18, 25, 26, 27, 28, 29, 69, 70, 71], "set": [1, 2, 5, 10, 12, 13, 14, 15, 18, 35, 36, 37, 40, 63, 69, 70, 71], "100": [1, 35, 70, 71], "pyproject": 1, "toml": 1, "instead": [1, 6, 10, 14, 15, 18, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 70, 71], "default": [1, 6, 10, 14, 15, 16, 18, 19, 24, 27, 31, 33, 35, 36, 37, 38, 40, 69, 70, 71], "88": [1, 71], "sure": [1, 3, 5, 14, 70, 71], "thorough": 1, "ani": [1, 3, 10, 12, 13, 14, 15, 18, 19, 24, 37, 69, 70, 71, 73], "should": [1, 5, 7, 10, 12, 13, 14, 18, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 36, 37, 63, 69, 70, 71], "do": [1, 3, 4, 5, 7, 10, 12, 13, 14, 18, 35, 37, 40, 69, 70, 71, 73], "directli": [1, 5, 12, 13, 15, 69, 70, 71], "automat": [1, 5, 7, 14, 15, 69, 70, 71], "gener": [1, 5, 7, 13, 14, 18, 31, 33, 35, 39, 40, 69, 70], "api": [1, 5, 37, 70], "when": [1, 3, 5, 7, 10, 11, 12, 13, 14, 15, 24, 31, 33, 35, 37, 38, 39, 40, 67, 69, 70, 71], "merg": [1, 5, 14], "main": [1, 3, 5, 6, 7, 10, 37, 70], "thei": [1, 4, 5, 14, 15, 18, 35, 40, 69, 70, 71, 73], "pytest": 1, "doctest": 1, "want": [1, 5, 7, 10, 14, 16, 35, 36, 37, 39, 69, 70, 71], "view": [1, 2], "chang": [1, 2, 3, 5, 14, 15, 37, 40, 69, 70, 71], "hot": [1, 70, 71], "reload": [1, 70, 71], "give": [1, 5, 10, 14, 15, 35, 38, 40, 69, 70, 71, 73], "real": [1, 7, 69, 70, 71, 73], "time": [1, 5, 7, 8, 10, 14, 36, 37, 69, 70, 71], "edit": [1, 7, 14, 40, 70, 71, 73], "googl": [1, 7, 38, 70, 71], "python": [1, 2, 15, 30, 35, 38, 69, 71, 72], "write": [1, 2, 3, 5, 14, 69, 70, 71, 73], "some": [1, 3, 5, 10, 14, 16, 18, 35, 37, 40, 69, 70], "from": [1, 2, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15, 16, 18, 24, 32, 35, 36, 38, 40, 69, 70, 71, 73], "restructuredtext": 1, "rest": [1, 5, 10, 14, 15, 35, 38, 69, 71], "In": [1, 2, 5, 6, 10, 12, 13, 14, 35, 37, 70, 71], "case": [1, 2, 5, 10, 14, 15, 35, 36, 37, 38, 40, 69, 70, 71, 72], "A": [1, 2, 4, 5, 10, 11, 12, 13, 14, 18, 24, 31, 32, 33, 35, 37, 39, 40, 69, 70, 71], "descript": 1, "what": [1, 3, 5, 7, 10, 14, 36, 37, 40, 71, 73], "doe": [1, 5, 10, 12, 13, 14, 15, 36, 37, 40, 69, 70, 71], "much": [1, 5, 10, 14, 35, 36, 40, 69, 70, 71], "detail": [1, 5, 10, 14, 15, 18, 24, 33, 37, 38, 40, 69, 70, 71], "necessari": [1, 5, 71], "fulli": [1, 40, 70], "understand": [1, 10, 14, 36, 71], "warn": [1, 10, 14, 36, 37, 69], "user": [1, 2, 5, 14, 15, 38, 69, 71], "e": [1, 5, 10, 12, 13, 14, 15, 18, 36, 37, 38, 69, 70, 71, 72], "g": [1, 5, 10, 12, 13, 14, 36, 37, 38, 69, 71], "common": [1, 5, 7, 10, 14, 15, 18, 69, 70, 71], "pitfal": 1, "exampl": [1, 2, 10, 12, 13, 14, 16, 18, 32, 35, 37, 69, 70], "here": [1, 2, 3, 5, 14, 15, 18, 19, 24, 35, 36, 69, 70, 71], "print": [1, 10, 35, 63, 69, 70, 71], "1": [1, 3, 4, 5, 10, 12, 13, 14, 15, 18, 19, 20, 24, 30, 31, 32, 36, 37, 38, 39, 69, 70, 71, 72], "2": [1, 3, 4, 10, 12, 13, 14, 15, 18, 35, 36, 38, 69, 70, 71, 72, 73], "3": [1, 6, 10, 11, 12, 13, 14, 15, 18, 35, 38, 40, 69, 70, 71, 72, 73], "arg": [1, 24, 37], "param_without_type_signatur": 1, "each": [1, 5, 10, 11, 12, 13, 14, 15, 18, 36, 37, 38, 39, 40, 67, 69, 70, 71], "indent": 1, "onc": [1, 3, 5, 14, 69, 70, 71], "more": [1, 5, 7, 10, 11, 14, 15, 18, 32, 36, 40, 69, 70, 71, 73], "param_2": 1, "anoth": [1, 5, 70, 71, 73], "paramet": [1, 5, 7, 10, 12, 13, 14, 15, 16, 18, 19, 24, 25, 26, 27, 31, 33, 35, 36, 37, 38, 40, 68, 69, 70], "return": [1, 5, 10, 11, 12, 13, 14, 18, 24, 27, 31, 33, 35, 36, 37, 38, 40, 63, 68, 69, 70, 71], "without": [1, 3, 5, 10, 14, 28, 29, 69, 70, 71], "type": [1, 6, 7, 10, 12, 13, 14, 15, 16, 21, 24, 27, 31, 32, 33, 36, 37, 38, 40, 63, 68, 69, 70, 71], "signatur": [1, 12, 13, 14, 71], "rais": [1, 14, 36, 38, 69, 71], "inform": [1, 14, 15, 32, 37, 38, 70, 71], "about": [1, 5, 7, 10, 14, 35, 37, 40, 69, 70, 71, 73], "error": [1, 10, 14, 36, 38, 71], "mai": [1, 5, 10, 12, 13, 14, 15, 18, 69, 70, 71], "part": [1, 5, 10, 14, 15, 26, 40, 70, 71, 73], "codebas": [1, 71], "cross": [1, 10, 14, 30, 31, 69, 70, 71], "referenc": [1, 5], "omit": [1, 36, 71], "full": [1, 4, 5, 10, 12, 14, 15, 18, 69, 71], "path": [1, 4, 69], "same": [1, 3, 5, 10, 11, 14, 15, 18, 36, 37, 39, 40, 67, 69, 70, 71], "mod": 1, "transformer_len": [1, 3, 5, 8, 70, 71], "modul": [1, 5, 8, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 23, 25, 26, 27, 28, 29, 31, 32, 33, 34, 35, 37, 38, 39, 40, 69, 71], "const": 1, "loading_from_pretrain": [1, 8, 9, 14, 71], "official_model_nam": [1, 14, 38], "hookedtransform": [1, 3, 5, 6, 7, 8, 9, 10, 12, 13, 15, 16, 35, 36, 38, 40, 63, 68, 69, 70, 71], "meth": [1, 10], "from_pretrain": [1, 3, 6, 10, 12, 13, 14, 16, 35, 38, 69, 70, 71], "attr": 1, "cfg": [1, 12, 13, 14, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 39, 44, 45, 51, 52, 58, 59, 60, 61, 62, 68, 69, 70, 71], "latex": 1, "re": [1, 3, 4, 7, 10, 14, 15, 36, 70, 71], "place": [1, 5, 7, 12, 13, 14, 69, 70, 71, 73], "string": [1, 12, 13, 14, 15, 35, 36, 37, 38, 69, 70, 71], "backward": [1, 37, 70, 71], "slash": 1, "must": [1, 10, 12, 13, 14, 15, 18, 36, 37, 66, 69, 70, 71], "repeat": [1, 14, 35, 39, 69, 70, 71], "inlin": 1, "displai": [1, 70, 71], "mode": [1, 10, 14, 15, 18, 26, 69, 70, 71], "b": [1, 11, 12, 14, 32, 35, 38, 40, 67, 69, 71, 72], "2ab": 1, "nowrap": 1, "begin": [1, 3, 5, 13, 14, 35, 36, 69, 70, 71], "eqnarrai": 1, "y": [1, 10, 14, 36, 70, 71], "ax": [1, 14, 40, 71], "bx": 1, "c": [1, 38, 69, 71, 72], "f": [1, 70, 71], "x": [1, 5, 10, 14, 18, 25, 26, 28, 29, 36, 37, 38, 66, 69, 70, 71], "2xy": 1, "end": [1, 5, 7, 14, 37, 40, 69, 70, 71], "ital": 1, "text": [1, 7, 10, 14, 15, 18, 31, 33, 35, 37, 38, 39, 69, 70], "bold": 1, "list": [1, 3, 4, 10, 12, 13, 14, 15, 35, 36, 37, 39, 40, 69, 70, 71], "item": [1, 5, 10, 69, 70, 71], "number": [1, 5, 10, 14, 15, 16, 18, 35, 38, 40, 63, 68, 69, 70, 71], "quot": 1, "level": [1, 37, 70, 71, 73], "extern": [1, 70], "link": [1, 14, 35], "domain": 1, "invalid": 1, "research": [2, 3, 4, 5, 7, 70, 71, 73], "done": [2, 4, 5, 6, 10, 14, 15, 18, 37, 70, 71], "involv": [2, 5, 70, 71], "progress": [2, 5, 14, 71], "measur": [2, 35, 36, 40, 69, 70], "grokk": [2, 7], "mechanist": [2, 3, 7, 40, 70, 71], "interpret": [2, 3, 7, 10, 14, 16, 36, 40, 69, 70], "iclr": 2, "spotlight": 2, "2023": 2, "lawrenc": 2, "chan": 2, "tom": [2, 70], "lieberum": 2, "jess": 2, "smith": 2, "jacob": 2, "steinhardt": 2, "find": [2, 5, 7, 10, 11, 14, 40, 70, 71], "neuron": [2, 7, 10, 14, 70, 71], "haystack": 2, "studi": [2, 4, 40, 70, 71], "spars": [2, 5], "probe": 2, "gurne": 2, "matthew": 2, "pauli": 2, "katherin": 2, "harvei": 2, "dmitrii": 2, "troitskii": 2, "dimitri": 2, "bertsima": 2, "toward": [2, 18, 40, 70], "autom": 2, "circuit": [2, 12, 14, 18, 35, 36, 40, 69, 70, 71], "discoveri": 2, "arthur": [2, 71], "conmi": [2, 71], "augustin": 2, "n": [2, 14, 18, 63, 66, 69, 70, 71], "mavor": 2, "parker": 2, "aengu": 2, "lynch": 2, "stefan": 2, "heimersheim": 2, "adri\u00e0": 2, "garriga": 2, "alonso": 2, "actual": [2, 5, 14, 36, 37, 71], "othello": [2, 7, 38, 72], "gpt": [2, 3, 4, 7, 10, 12, 13, 14, 15, 18, 19, 24, 35, 38, 69, 70, 71, 72, 73], "ha": [2, 3, 4, 5, 10, 11, 12, 13, 14, 18, 24, 35, 38, 39, 40, 67, 69, 70, 71], "linear": [2, 7, 12, 13, 14, 18, 38, 67, 70, 71], "emerg": [2, 7], "world": [2, 5, 7, 71, 73], "represent": [2, 7], "docstr": 2, "4": [2, 3, 5, 15, 18, 35, 69, 70, 71, 72], "layer": [2, 6, 10, 12, 13, 14, 15, 16, 18, 19, 21, 24, 25, 26, 36, 37, 38, 39, 40, 67, 68, 69, 71], "attent": [2, 7, 10, 12, 13, 14, 15, 17, 18, 20, 24, 27, 30, 31, 33, 36, 40, 64, 69, 71], "transform": [2, 3, 4, 7, 10, 12, 13, 14, 15, 18, 19, 20, 21, 22, 23, 25, 26, 27, 28, 29, 31, 32, 33, 34, 38, 39, 40, 66, 69, 70], "jett": 2, "janiak": 2, "toi": [2, 14], "univers": 2, "icml": 2, "bilal": 2, "chughtai": 2, "n2g": 2, "scalabl": 2, "approach": [2, 5, 10, 70, 71], "quantifi": [2, 36], "larg": [2, 5, 7, 15, 18, 38, 69, 71, 72, 73], "languag": [2, 12, 14, 35, 63, 69, 70, 71], "workshop": 2, "rtml": 2, "alex": [2, 71], "foot": [2, 14, 71], "esben": 2, "kran": 2, "ioanni": 2, "konsta": 2, "fazl": 2, "barez": 2, "elicit": 2, "latent": 2, "predict": [2, 7, 10, 12, 14, 22, 35, 36, 69, 70, 71], "tune": [2, 12, 13, 38, 69, 71, 72], "len": [2, 10, 38], "nora": 2, "belros": 2, "zach": 2, "furman": 2, "logan": 2, "danni": 2, "halawi": 2, "igor": 2, "ostrovski": 2, "lev": 2, "mckinnei": 2, "stella": 2, "biderman": 2, "contribut": [2, 5, 10, 14, 70], "being": [2, 5, 10, 12, 13, 14, 15, 36, 37, 40, 69, 70, 71], "induct": [2, 4, 35, 36, 38], "head": [2, 4, 7, 10, 12, 13, 14, 15, 16, 18, 22, 24, 35, 36, 38, 40, 67, 69], "phase": 2, "replic": [2, 4, 14, 16, 35, 70, 71], "partial": [2, 70, 71], "context": [2, 10, 14, 37, 40, 69, 70, 71], "learn": [2, 3, 7, 15, 63, 69, 70, 71, 73], "connor": 2, "kissan": 2, "decis": [2, 3], "script": [2, 7], "train": [2, 7, 8, 9, 10, 12, 13, 14, 15, 35, 38, 69, 70, 73], "which": [2, 3, 5, 6, 7, 10, 12, 13, 14, 15, 35, 36, 37, 38, 39, 40, 69, 70, 71, 73], "intermedi": [2, 10, 14, 37, 71], "activ": [2, 3, 4, 5, 7, 10, 12, 13, 14, 15, 16, 37, 40, 65, 69, 73], "perform": [2, 6, 7, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 35, 37, 69, 70, 71], "attribut": [2, 4, 7, 10, 18, 40, 69, 71], "ablat": [2, 70, 71], "up": [2, 3, 4, 5, 10, 14, 15, 37, 40, 63, 69, 70, 71], "initi": [2, 5, 8, 14, 15, 24, 37, 68, 69, 70, 71], "work": [2, 3, 4, 5, 7, 10, 12, 13, 14, 18, 37, 38, 69, 70, 71, 73], "found": [2, 3, 5, 6, 14, 15, 70, 71], "demo": [3, 5, 16, 38, 72, 73], "how": [3, 5, 7, 10, 14, 36, 40, 63, 70, 71, 73], "basic": [3, 7, 14, 35, 69, 70], "To": [3, 4, 5, 10, 14, 15, 18, 37, 70, 71], "see": [3, 5, 7, 10, 12, 13, 14, 15, 18, 24, 32, 33, 36, 37, 38, 40, 69, 70, 71, 73], "exploratori": [3, 7, 36, 69, 71, 73], "analysi": [3, 7, 10, 14, 36, 69, 71, 73], "practic": [3, 4, 7, 70, 71], "look": [3, 4, 5, 7, 8, 10, 14, 18, 36, 40, 69, 70, 71, 73], "out": [3, 5, 7, 10, 14, 16, 40, 69, 70, 71], "my": [3, 5, 7, 14, 15, 69, 70, 71, 73], "analys": [3, 7, 10, 14, 71], "indirect": [3, 4, 7, 35], "object": [3, 4, 5, 7, 10, 11, 12, 13, 14, 15, 16, 35, 37, 38, 39, 63, 69], "identif": [3, 4, 7, 35], "record": [3, 7, 71], "myself": [3, 5, 7, 71], "veri": [3, 4, 5, 7, 10, 15, 16, 35, 70, 71, 73], "young": [3, 4, 71], "small": [3, 4, 5, 6, 7, 10, 14, 15, 35, 38, 69, 70, 71, 72, 73], "field": [3, 4, 5, 14, 69, 71, 73], "lot": [3, 4, 5, 7, 10, 11, 39, 40, 69, 70, 71, 73], "open": [3, 4, 5, 14, 35, 73], "problem": [3, 4, 5, 71, 73], "would": [3, 4, 5, 12, 18, 32, 70, 71, 73], "help": [3, 4, 5, 15, 40, 70, 71, 73], "try": [3, 4, 10, 14, 36, 70, 71], "concret": [3, 4, 70, 71], "figur": [3, 40, 70, 71], "where": [3, 5, 6, 10, 11, 12, 13, 14, 15, 18, 33, 36, 37, 38, 40, 63, 69, 70, 71], "skill": [3, 71], "kei": [3, 4, 5, 10, 12, 13, 14, 15, 18, 19, 24, 30, 31, 33, 35, 36, 39, 40, 69, 70, 71], "resourc": [3, 4, 5], "new": [3, 7, 10, 14, 37, 38, 39, 69, 70, 71], "tutori": [3, 4, 5, 70, 71], "scratch": [3, 4, 70], "an": [3, 4, 7, 10, 11, 12, 13, 14, 15, 18, 35, 36, 37, 38, 39, 40, 63, 67, 69, 70, 73], "accompani": [3, 4, 7, 71], "templat": [3, 35], "yourself": [3, 14, 70, 71], "One": [3, 5, 14, 70, 71, 73], "signific": [3, 37, 70, 71], "design": [3, 5, 10, 70, 71, 73], "made": [3, 5, 35, 70, 71], "wa": [3, 5, 6, 10, 12, 13, 14, 15, 35, 40, 70, 71], "singl": [3, 5, 10, 12, 14, 18, 31, 32, 33, 39, 40, 69, 70, 71], "implement": [3, 5, 12, 13, 14, 18, 40, 66, 69, 70, 71], "could": [3, 5, 70, 71], "support": [3, 5, 7, 12, 13, 14, 15, 30, 36, 37, 65, 66, 69, 70, 71], "rang": [3, 4, 5, 14, 16, 36, 40, 69, 70, 71], "subtli": [3, 18], "differ": [3, 5, 6, 10, 12, 13, 14, 15, 18, 35, 36, 37, 40, 69, 70, 71], "style": [3, 5, 10, 12, 13, 14, 15, 19, 36, 70, 71, 73], "upsid": 3, "just": [3, 4, 5, 10, 14, 15, 35, 40, 69, 70, 71], "arbitrari": [3, 14, 70, 71], "name": [3, 5, 10, 14, 15, 35, 36, 37, 38, 40, 63, 69], "But": [3, 10, 14, 40, 69, 70, 71], "downsid": 3, "py": [3, 5, 12, 13, 66], "compon": [3, 8, 9, 10, 12, 13, 14, 15, 67, 69, 70, 71], "difficult": [3, 10], "recommend": [3, 8, 10, 14, 15, 16, 37, 70, 71], "clean": [3, 40, 69, 70, 71], "minim": [3, 5, 71], "intern": [3, 5, 10, 14, 40, 70, 71, 73], "architectur": [3, 12, 13, 70], "significantli": [3, 12, 13, 14, 35, 40, 70, 71], "clearer": 3, "better": [3, 14, 15, 35, 36, 38, 70, 71], "document": [3, 14, 69, 71], "pip": [3, 5, 70, 71], "git": 3, "import": [3, 5, 10, 14, 16, 35, 39, 40, 69, 73], "known": [3, 73], "easytransform": [3, 71, 73], "break": [3, 5, 10, 70, 71], "been": [3, 5, 10, 14, 69, 71], "sinc": [3, 5, 10, 14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 70, 71], "renam": [3, 5], "old": [3, 5, 38, 71], "version": [3, 7, 14, 35, 37, 66, 70, 71], "legaci": [3, 36], "run": [3, 5, 6, 10, 14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 38, 39, 40, 63, 70, 73], "v1": 3, "avail": [3, 5, 7, 10, 14, 15, 36, 38], "requir": [3, 5, 12, 13, 14, 40, 69, 71], "luckili": 3, "provid": [3, 10, 12, 13, 14, 25, 28, 32, 37, 68, 69], "wai": [3, 5, 6, 10, 14, 15, 37, 69, 70, 71], "those": [3, 5, 6, 14, 37, 69, 70], "configur": [3, 5, 15, 63, 68], "environment": 3, "variabl": [3, 12, 13], "simpli": [3, 5, 70], "token": [3, 7, 10, 12, 13, 14, 15, 16, 18, 19, 21, 22, 23, 24, 27, 31, 32, 33, 35, 36, 38, 40, 69, 70], "hf_token": 3, "agreement": 3, "issu": [3, 5, 14, 70, 71], "attempt": [3, 5, 14], "ue": 3, "befor": [3, 10, 12, 13, 14, 15, 18, 19, 20, 37, 69, 70, 71], "relat": [3, 14, 18, 37, 70, 71], "consol": 3, "output": [3, 5, 7, 10, 12, 13, 14, 15, 18, 36, 37, 40, 70, 71], "point": [3, 5, 10, 12, 13, 14, 15, 16, 33, 37, 69, 70, 73], "As": [3, 14, 15, 69, 70, 71], "23": [3, 70, 71], "24": [3, 14, 69, 70, 71, 72], "current": [3, 5, 10, 12, 13, 14, 15, 18, 19, 24, 36, 70, 71], "co": [3, 69], "mistralai": [3, 38], "mixtral": [3, 5, 6, 38, 42, 72], "8x7b": [3, 38], "v0": [3, 38, 72], "mistral": [3, 15, 18, 35, 38, 42, 72], "7b": [3, 5, 7, 38, 71, 72], "instruct": [3, 38, 71, 72], "mean": [4, 5, 10, 14, 15, 16, 18, 19, 24, 28, 29, 36, 37, 69, 70, 71], "": [4, 5, 7, 10, 11, 12, 13, 14, 15, 16, 18, 35, 36, 37, 38, 40, 68, 69, 71, 73], "both": [4, 5, 10, 14, 15, 18, 36, 37, 39, 66, 70, 71], "low": [4, 11, 14, 15, 18, 69, 71], "hang": [4, 71], "fruit": [4, 71], "bar": [4, 14], "entri": [4, 18, 39, 40, 71], "The": [4, 5, 7, 10, 11, 12, 13, 14, 15, 16, 18, 19, 22, 24, 27, 31, 32, 33, 35, 36, 37, 38, 39, 40, 63, 68, 69, 70, 71, 73], "standard": [4, 6, 14, 15, 70, 71], "answer": [4, 10, 40, 69, 70, 71], "why": [4, 5, 10, 18, 69, 70, 71], "yet": [4, 5, 12, 13, 14, 70, 71, 73], "aren": [4, 38, 71], "t": [4, 5, 10, 11, 12, 13, 14, 15, 16, 18, 35, 38, 69, 70, 71, 73], "enough": [4, 5, 10, 70, 71], "peopl": [4, 5, 71], "guid": [4, 71], "arena": 4, "callum": [4, 71], "mcdougal": [4, 71], "comprehens": [4, 71], "introduct": 4, "mech": [4, 70], "interp": [4, 70], "written": [4, 5, 7, 70], "snippet": 4, "copi": [4, 12, 13, 70], "come": [4, 5, 14, 15, 40, 70, 71], "exercis": [4, 70], "solut": [4, 5, 70, 71], "notabl": [4, 14, 37, 70, 71], "video": [4, 7, 70, 71], "me": [4, 5, 38, 71, 73], "good": [4, 5, 7, 10, 35, 69, 70, 71, 73], "cover": [4, 14, 71], "foundat": [4, 71], "concept": [4, 70, 71], "wild": [4, 10, 70, 71], "techniqu": [4, 7, 14, 40, 70, 71], "direct": [4, 5, 10, 14, 16, 37, 40, 71], "logit": [4, 5, 6, 7, 10, 12, 13, 14, 15, 22, 35, 40, 69, 71], "patch": [4, 5, 7, 8, 9], "paper": [4, 7, 10, 14, 15, 18, 32, 35, 40, 71], "read": [4, 5, 7, 10, 14, 71], "200": [4, 71], "explain": [4, 7, 70, 71], "jargon": 4, "unfamiliar": [4, 70], "term": [4, 10, 14, 70], "go": [4, 5, 7, 40, 71, 73], "across": [4, 5, 10, 12, 13, 14, 16, 38, 40, 68, 70, 71], "youtub": 4, "channel": 4, "content": [4, 35, 70, 71], "walkthrough": [4, 70, 71], "am": 5, "happi": 5, "announc": 5, "now": [5, 7, 14, 15, 70, 71], "releas": 5, "recent": 5, "primari": 5, "motiv": [5, 70], "behind": [5, 18, 70], "jump": [5, 71], "transit": [5, 70], "strictli": [5, 10, 71], "describ": [5, 15, 69, 70], "At": [5, 70], "last": [5, 10, 14, 69, 71], "minut": 5, "did": [5, 10, 69, 70, 71], "remov": [5, 10, 11, 14, 18, 37, 69, 70, 71, 73], "hookedsa": 5, "had": [5, 70, 71], "saelen": 5, "bundl": [5, 71], "major": 5, "hand": [5, 70, 71], "modif": 5, "affect": [5, 14, 15, 40, 70], "bryce": 5, "meyer": 5, "softwar": 5, "engin": [5, 40, 70, 71, 73], "littl": [5, 15, 71, 73], "under": [5, 10, 14, 24], "15": [5, 70, 71], "profession": [5, 7], "experi": [5, 7, 15, 70, 71, 73], "wide": 5, "expertis": 5, "embed": [5, 7, 10, 12, 13, 14, 15, 18, 21, 22, 27, 33, 70, 71], "comput": [5, 10, 11, 14, 20, 21, 22, 23, 25, 26, 28, 29, 30, 32, 34, 36, 37, 39, 40, 69, 70, 71, 73], "coupl": 5, "gotten": [5, 14], "ml": [5, 70, 71, 73], "especi": [5, 14, 70, 71], "ai": [5, 15, 18, 38, 69, 71], "safeti": 5, "nine": 5, "march": 5, "chat": [5, 38, 71, 72], "bit": [5, 15, 70, 71], "he": [5, 71], "ask": 5, "might": [5, 10, 12, 13, 70], "interest": [5, 7, 12, 13, 14, 70, 71], "take": [5, 7, 10, 14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 40, 69, 70, 71, 73], "role": 5, "maintain": 5, "basi": [5, 12, 14], "april": 5, "far": [5, 14, 69, 70, 71], "pretti": [5, 10, 14, 69, 70, 71], "mani": [5, 14, 24, 39, 40, 63, 70, 71], "kind": [5, 10, 70, 71], "address": 5, "everi": [5, 10, 12, 13, 14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 40, 63, 69, 70, 71], "await": 5, "repli": 5, "total": [5, 70, 71], "around": [5, 6, 10, 12, 13, 14, 37, 40, 68, 70, 71], "30": [5, 70, 71, 72], "20": [5, 69, 70, 71, 72], "pr": 5, "were": [5, 10, 14, 15, 35, 69, 70, 71, 73], "limit": [5, 12, 13, 14, 70], "llama": [5, 7, 15, 38, 42, 72], "quantiz": [5, 15], "hookedsaetransform": 5, "brand": 5, "class": [5, 10, 11, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 37, 38, 39, 63, 69], "splice": 5, "autoencod": 5, "two": [5, 6, 11, 12, 32, 36, 38, 40, 69, 70, 71], "goal": [5, 70, 71, 73], "posit": [5, 7, 10, 12, 13, 14, 15, 18, 21, 27, 30, 33, 35, 36, 37, 38, 40, 69, 70, 71], "while": [5, 12, 13, 14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 70, 71], "remain": [5, 10, 14, 37, 71], "power": [5, 71], "who": 5, "push": 5, "second": [5, 14, 35, 70, 71], "base": [5, 7, 10, 11, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 37, 38, 39, 63, 68, 69, 70, 71, 72], "easier": [5, 14, 35, 70, 71, 73], "futur": 5, "llm": 5, "continu": [5, 40, 70, 71], "acceler": [5, 70, 71], "feel": [5, 70, 71, 73], "massiv": [5, 70, 71], "amount": [5, 10, 15, 70], "momentum": [5, 63], "moment": [5, 18, 19, 24], "hope": [5, 71], "carri": 5, "over": [5, 10, 14, 40, 69, 70, 71], "background": [5, 70], "know": [5, 7, 10, 12, 13, 70, 71], "talk": [5, 71], "ensur": [5, 14, 70], "meet": 5, "person": [5, 70], "spoken": 5, "dozen": 5, "commun": 5, "happen": [5, 70, 71], "appoint": 5, "curiou": 5, "hear": 5, "anyon": [5, 35], "tool": [5, 7, 71, 73], "absolut": [5, 6, 12, 13, 14, 15, 18, 27, 36, 69, 70, 71], "beginn": 5, "complet": [5, 10, 18, 69, 70, 71], "expert": [5, 15], "Not": [5, 18, 19, 24, 26], "idea": [5, 14, 18, 40, 70, 71, 73], "evolv": 5, "biggest": [5, 10], "previous": [5, 70], "offici": [5, 38, 71], "instanc": [5, 12, 13, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 36, 37, 70], "compat": [5, 14, 38, 73], "through": [5, 14, 37, 70, 71], "forward": [5, 10, 12, 13, 14, 15, 18, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 37, 39, 71], "start": [5, 10, 13, 14, 15, 18, 40, 69, 70, 71, 73], "todai": [5, 71, 73], "assur": 5, "abl": [5, 70, 71], "upgrad": 5, "worri": 5, "There": [5, 6, 10, 12, 13, 36, 38, 69, 70, 71, 73], "right": [5, 12, 13, 14, 18, 19, 40, 69, 70, 71], "move_model": [5, 10], "activationcach": [5, 8, 9, 12, 13, 14, 36, 40, 69, 70, 71], "function": [5, 10, 12, 13, 14, 15, 18, 20, 21, 22, 23, 24, 25, 26, 28, 29, 32, 34, 36, 37, 38, 40, 63, 65, 66, 68, 69, 71, 73], "cache_al": [5, 37], "hook_point": [5, 8, 9, 14, 71], "keep": [5, 10, 14, 37, 70, 71, 73], "thing": [5, 11, 14, 15, 18, 40, 70, 71, 73], "simpl": [5, 70, 71], "howev": [5, 10, 14, 24, 35, 70, 71], "them": [5, 10, 14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 40, 69, 70, 71], "adapt": [5, 71], "awai": [5, 14, 70], "along": [5, 14, 18, 69, 70, 71], "anyth": [5, 35, 70], "mark": [5, 70, 71], "whenev": [5, 37], "someth": [5, 14, 35, 70], "becom": [5, 70, 71], "promin": 5, "sort": [5, 37, 71], "slip": 5, "scenario": 5, "situat": 5, "handl": [5, 14, 37], "persist": [5, 37], "allow": [5, 14, 36, 40, 69, 70, 71], "reli": 5, "interrupt": 5, "still": [5, 37, 70], "encourag": [5, 18, 69], "everyon": 5, "period": [5, 70, 71], "ey": 5, "don": [5, 10, 12, 13, 14, 15, 16, 35, 69, 70, 71, 73], "imagin": [5, 70], "often": [5, 10, 14, 15, 38, 69, 70, 71], "save": [5, 10, 14, 15, 36, 63, 69, 70, 71], "troubl": [5, 10], "move": [5, 10, 12, 13, 14, 40, 70, 71], "three": [5, 14, 40, 69, 70], "timefram": 5, "plan": 5, "state": [5, 10, 14, 31, 37, 70, 71, 73], "tracker": 5, "categor": 5, "easi": [5, 10, 14, 69, 70, 71, 73], "date": [5, 14], "below": [5, 12, 13, 14, 70], "draft": 5, "our": [5, 33, 70, 71, 73], "priorit": 5, "feedback": [5, 69, 70, 71, 73], "surfac": 5, "other": [5, 10, 12, 14, 18, 19, 24, 36, 37, 38, 40, 70], "improv": [5, 69, 70, 71], "achiev": [5, 10, 71], "diagnos": 5, "variou": [5, 15, 37, 70, 71, 73], "area": 5, "memori": [5, 10, 11, 12, 13, 14, 15, 70, 71], "leak": 5, "occur": [5, 70], "seem": [5, 14, 15, 35, 38, 70, 71], "refer": [5, 10, 14, 19, 37, 70, 71], "properli": [5, 35, 70], "thu": 5, "caus": 5, "garbag": 5, "collect": [5, 71], "correctli": [5, 14], "identifi": [5, 7, 14, 40, 70, 71], "proper": 5, "overal": [5, 15, 70], "deal": [5, 14, 18, 69, 70], "larger": [5, 10, 35, 70, 71], "task": [5, 7, 12, 14, 15, 35, 40, 63, 70], "explor": [5, 69, 71], "abil": [5, 10, 40, 70], "batch": [5, 10, 12, 13, 14, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 31, 32, 33, 34, 35, 37, 39, 40, 63, 66, 67, 69, 70, 71], "process": [5, 14, 15, 38, 70, 71], "alreadi": [5, 10, 14, 69, 70, 71], "share": 5, "well": [5, 7, 14, 16, 37, 40, 69, 70, 71], "togeth": [5, 14, 69, 70, 71], "separ": [5, 12, 13, 14, 15, 37, 67, 69, 70, 71], "volunt": 5, "said": [5, 71], "submiss": 5, "discuss": [5, 70], "think": [5, 14, 35, 69, 70, 71], "few": [5, 10, 12, 13, 70, 71], "week": 5, "confus": [5, 70, 71], "among": 5, "calcul": [5, 10, 11, 12, 13, 14, 15, 18, 24, 36, 67, 69, 70, 71], "match": [5, 10, 14, 36, 66, 70], "huggingfac": [5, 12, 13, 14, 15, 35, 38, 66, 69, 71, 73], "solv": [5, 37, 70, 71, 73], "systemat": 5, "submit": 5, "show": [5, 7, 14, 16, 36, 70, 71, 73], "order": [5, 14, 18, 38, 40, 69, 70], "allevi": 5, "build": [5, 37, 71, 73], "spit": 5, "tabl": [5, 70, 71], "u": [5, 11, 14, 40, 69, 70, 71], "snapshot": 5, "store": [5, 10, 14, 15, 24, 37, 39, 40, 63, 70, 71], "repo": [5, 70], "regener": 5, "cumul": [5, 14, 69], "valu": [5, 7, 10, 11, 12, 13, 14, 15, 18, 24, 30, 31, 33, 36, 38, 39, 40, 69, 70, 71, 73], "creat": [5, 7, 10, 14, 18, 37, 70, 71], "robust": [5, 10, 70], "big": [5, 36, 38, 69, 70, 71], "famili": [5, 15, 71], "hard": [5, 14, 70, 71], "even": [5, 7, 11, 14, 15, 18, 35, 38, 70, 71, 73], "smallest": [5, 71], "thought": [5, 10, 70, 71], "thrown": 5, "topic": [5, 7], "best": [5, 7, 14, 70, 71], "guess": [5, 70], "reason": [5, 18, 19, 24, 70, 71], "untrain": 5, "eg": [5, 10, 14, 35, 40, 69, 70, 71], "randomli": [5, 14, 15, 71], "weight": [5, 7, 12, 13, 14, 15, 18, 24, 26, 63, 66, 69, 70, 71, 73], "verifi": [5, 7, 70, 71], "load": [5, 10, 12, 13, 14, 15, 35, 38, 69, 70, 73], "result": [5, 10, 12, 13, 14, 15, 21, 30, 33, 36, 38, 40, 69, 70, 71, 73], "accur": 5, "sens": [5, 11, 37, 70, 71], "consist": [5, 14, 70, 71], "sampl": [5, 14, 35, 69], "size": [5, 10, 14, 15, 18, 35, 63, 69, 70, 71], "against": [5, 70], "bite": 5, "success": [5, 70], "turn": [5, 10, 14, 69, 70, 71], "effici": [5, 11, 18, 69, 71], "proof": [5, 70], "put": [5, 70, 71], "strong": 5, "opinion": 5, "most": [5, 10, 14, 37, 69, 70, 71, 73], "roundtabl": 5, "wrapper": [5, 10, 12, 13, 14, 40, 68, 71], "plugin": 5, "addit": [5, 7, 12, 13, 14, 70, 71], "outsid": 5, "publish": 5, "themselv": [5, 14], "final": [5, 6, 10, 12, 13, 14, 15, 18, 69, 70, 71], "overhaul": 5, "composit": [5, 14, 70, 71], "util": [5, 8, 9, 10, 11, 14, 36, 37, 38, 63, 70, 71], "isol": [5, 70], "rapidli": 5, "itself": [5, 36, 69, 70], "none": [5, 10, 12, 13, 14, 15, 16, 18, 19, 20, 21, 24, 25, 27, 28, 30, 31, 33, 35, 36, 37, 38, 40, 63, 68, 69, 70, 71], "pain": [5, 71], "rel": [5, 15, 30, 70], "grow": 5, "exponenti": 5, "whole": [5, 70, 71], "explod": 5, "section": [5, 15, 70, 71], "relev": [5, 14, 15, 18, 40, 69, 70, 71], "skip": [5, 14, 70, 71], "setup": [5, 14, 37, 40], "act": [5, 37, 40, 69, 70, 71], "vast": 5, "due": [5, 6, 14, 71], "potenti": 5, "mismatch": [5, 36], "between": [5, 10, 14, 15, 18, 36, 40, 69, 70, 71, 73], "meant": 5, "repres": [5, 11, 12, 15, 32, 36, 40, 69, 70, 71], "updat": [5, 7, 14, 39, 40, 68, 70, 71], "readi": 5, "sent": 5, "justifi": 5, "bug": [5, 7, 10, 15, 71], "fix": [5, 37, 70, 71], "exist": [5, 14, 36, 38, 70, 71], "split": [5, 14, 18, 38, 69, 70, 71], "group": [5, 14, 15, 18, 24], "call": [5, 10, 12, 13, 14, 15, 20, 21, 22, 23, 24, 25, 26, 28, 29, 32, 34, 37, 38, 69, 70, 71], "again": [5, 70], "piec": [5, 70], "tradition": 5, "everyth": [5, 37, 40, 69, 71], "That": [5, 70], "mock": 5, "spi": 5, "control": [5, 15, 40, 70, 71], "input": [5, 10, 12, 13, 14, 15, 24, 27, 32, 37, 38, 39, 40, 66, 67, 69, 70, 71], "side": [5, 11, 14], "effect": [5, 10, 14, 15, 40, 70, 71], "certain": [5, 15, 40], "logic": 5, "entir": [5, 10, 14, 40, 70], "rule": 5, "incredibli": [5, 71], "cannot": [5, 14, 69, 71], "origin": [5, 15, 16, 18, 70, 71], "pass": [5, 10, 14, 15, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 32, 33, 34, 36, 37, 38, 39, 69, 71], "bunch": [5, 10, 12, 13, 14, 37, 70, 71], "Being": [5, 70], "live": [5, 12, 13, 14, 70, 71], "ci": 5, "report": 5, "great": [5, 7, 71, 73], "get": [5, 7, 10, 12, 13, 14, 15, 16, 18, 35, 36, 37, 40, 68, 70, 71, 73], "review": 5, "meaning": [5, 15, 40], "than": [5, 6, 10, 14, 15, 18, 20, 35, 36, 37, 40, 69, 70, 71], "ever": [5, 69], "substanti": 5, "let": [5, 15, 69, 70, 71, 73], "shift": [5, 40, 71], "favor": 5, "individu": [5, 7, 10, 14, 18, 70], "3000": 5, "18": [5, 69, 70, 71, 72], "distinct": 5, "either": [5, 14, 36, 37, 38, 40, 70, 71], "interdepend": 5, "least": [5, 40, 71], "depend": [5, 10, 14, 71], "its": [5, 10, 14, 18, 37, 70, 71, 73], "own": [5, 7, 70, 71], "realli": [5, 35, 36, 69, 70, 71], "anywher": [5, 40], "els": [5, 14, 15, 18, 38, 69, 70, 71], "mlp": [5, 10, 12, 13, 14, 15, 20, 33, 40, 69, 70, 71], "exactli": [5, 14, 38, 66, 70, 71], "thank": [5, 71], "excit": 5, "standpoint": 5, "perspect": [5, 71], "comparison": [5, 36], "worth": [5, 7, 10, 70, 71], "enabl": [5, 10, 14, 70, 71, 73], "huge": 5, "impact": [5, 10], "bring": 5, "realiti": [5, 70], "semver": 5, "older": [5, 71], "log": [5, 14, 63, 69, 70, 71], "data": [5, 6, 7, 14, 35, 69, 70, 71], "expos": [5, 14, 73], "properti": [5, 11, 12, 13, 14, 18, 24, 69, 70, 71], "minor": [5, 70], "bump": [5, 71], "whatsoev": 5, "With": [5, 18, 71], "fact": [5, 14, 70, 71, 73], "discov": 5, "earlier": [5, 70, 71], "extent": 5, "probabl": [5, 7, 14, 35, 36, 40, 69, 70, 71], "regardless": [5, 69], "stand": 5, "reliabl": 5, "17": [5, 70, 71], "possibli": 5, "easiest": [5, 70], "fresh": 5, "consum": [5, 10, 11], "top": [6, 14, 69, 70, 71], "k": [6, 10, 11, 12, 13, 14, 16, 18, 24, 40, 69, 70, 71], "gate": [6, 14], "hidden": [6, 15, 31, 71], "amplifi": 6, "greatli": [6, 71, 73], "select": [6, 10, 69, 70, 71], "lead": [6, 7, 12, 13, 37, 69, 71], "higher": [6, 14, 70], "normal": [6, 10, 14, 15, 26, 69, 70, 71, 73], "varianc": [6, 70], "test": [6, 7, 14, 35, 36, 69, 70, 71], "half": [6, 11, 12, 13, 14, 18, 35, 71], "precis": [6, 36, 40, 70, 71], "deviat": [6, 15, 71], "compar": [6, 35, 69, 71, 73], "2e": 6, "mitig": 6, "disabl": [6, 14, 36, 37, 70], "preprocess": [6, 12, 13, 70], "option": [6, 10, 12, 13, 14, 15, 16, 18, 19, 20, 21, 24, 25, 27, 28, 30, 31, 33, 35, 36, 37, 38, 39, 40, 63, 68, 69], "from_pretrained_no_process": [6, 14], "increas": [6, 40, 70, 71], "colab": [7, 70, 71, 73], "blob": [7, 66], "ipynb": 7, "causal": [7, 12, 15, 40, 70, 71], "intervent": [7, 40, 70, 71], "matter": [7, 14, 40, 70, 71], "produc": [7, 14, 40, 70], "incomplet": 7, "gradient": [7, 10, 37, 63, 71], "approxim": [7, 70, 71], "bad": [7, 14], "residu": [7, 10, 12, 13, 14, 15, 18, 31, 33, 34, 40, 71], "stream": [7, 10, 12, 13, 14, 15, 31, 33, 40, 69, 71], "after": [7, 10, 14, 15, 20, 26, 37, 63, 70, 71, 73], "demonstr": [7, 16, 70, 71], "focus": [7, 70, 71], "less": [7, 14, 18, 70, 71], "rigor": [7, 70, 71], "grasp": 7, "steal": 7, "liber": [7, 37], "phenomenon": 7, "memoris": 7, "minimis": 7, "loss": [7, 10, 14, 35, 37, 40, 63, 69, 70, 71], "longer": 7, "generalis": [7, 70, 71], "sharp": [7, 71], "decreas": [7, 18, 69, 70], "modular": [7, 69], "grok": 7, "light": 7, "explan": [7, 40, 70], "ll": [7, 14, 36, 70, 71], "pair": [7, 11, 14, 18, 36, 69, 70, 71], "seri": [7, 10, 71], "detector": [7, 36], "detect": [7, 36, 70, 71], "sever": [7, 10, 14, 69, 70, 71], "custom": [7, 14, 15, 21, 35, 37, 69, 70, 71], "algorithm": [7, 11, 15, 71, 73], "interact": [7, 65, 70, 71], "neuroscop": [7, 71], "hacki": [7, 69], "web": [7, 71], "visualis": [7, 70], "front": 7, "visual": [7, 12, 13, 71], "dynam": [7, 15, 71], "convert": [7, 12, 13, 14, 38, 69, 70, 71], "meta": [7, 14, 38, 69, 70, 71, 72], "until": [7, 10, 14, 37, 70, 71], "multi": [7, 10, 69, 71], "gpu": [7, 10, 11, 12, 13, 14, 70, 71], "access": [7, 10, 15, 37, 69, 70], "No": [7, 71], "previou": [7, 10, 14, 31, 33, 36, 70, 71], "port": 7, "excel": [7, 10, 70, 71, 73], "sequenc": [7, 12, 13, 14, 15, 18, 32, 35, 36, 37, 40, 69, 70, 71], "investig": [7, 10, 14, 36, 70, 71], "svd": [7, 11, 14, 16, 71], "conjectur": 7, "post": [7, 10, 15, 16, 70, 71], "singular": [7, 11, 14, 16, 71], "decomposit": [7, 10, 11, 14, 70, 71], "matric": [7, 11, 12, 13, 14, 16, 18, 19, 24, 66, 70, 71, 73], "surprisingli": 7, "reproduc": [7, 15, 36], "further": [7, 10, 14, 69, 70, 71], "tracr": 7, "cool": 7, "deepmind": 7, "compil": 7, "program": [7, 71, 73], "rasp": 7, "jax": 7, "form": [7, 10, 11, 14, 40, 70, 71], "pytorch": [7, 14, 15, 35, 37, 71], "brows": 8, "first": [8, 10, 14, 15, 35, 38, 40, 69, 70, 71], "submodul": 8, "factoredmatrix": [8, 9, 12, 13, 18, 69, 71], "hookedencod": [8, 9, 68], "hookedencoderdecod": [8, 9, 68], "hookedtransformerconfig": [8, 9, 14, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 38, 39, 44, 45, 51, 52, 58, 59, 60, 61, 62, 68], "svdinterpret": [8, 9], "eval": [8, 9, 71], "head_detector": [8, 9], "past_key_value_cach": [8, 9], "subpackag": 8, "pretrain": [8, 9, 10, 12, 13, 14, 15, 35, 38, 69, 70, 71], "core": [10, 14, 70, 71, 73], "varieti": [10, 71], "helper": [10, 14, 18, 35, 37, 40, 69, 71], "skim": 10, "method": [10, 12, 13, 14, 15, 37, 38, 39, 69, 70, 71], "back": [10, 15, 18, 19, 24, 71], "cache_dict": 10, "dict": [10, 12, 13, 14, 15, 18, 19, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 32, 33, 34, 35, 36, 37, 38, 69], "str": [10, 12, 13, 14, 15, 18, 19, 24, 30, 35, 36, 37, 38, 39, 40, 63, 68, 69, 70, 71], "tensor": [10, 11, 12, 13, 14, 16, 18, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 36, 37, 39, 40, 66, 67, 69, 70, 71, 73], "has_batch_dim": 10, "bool": [10, 12, 13, 14, 15, 30, 31, 35, 36, 37, 38, 39, 40, 63, 69], "run_with_cach": [10, 12, 13, 14, 37, 70, 71], "particular": [10, 16, 36, 70, 71], "behaviour": [10, 14, 70, 71], "modal": 10, "step": [10, 14, 15, 38, 63, 69, 70, 71], "respons": [10, 15, 70], "prompt": [10, 14, 18, 35, 40, 69, 70, 71], "chicken": 10, "road": [10, 69], "specif": [10, 12, 13, 14, 18, 36, 38, 40, 70, 71], "sublay": 10, "commonli": 10, "fall": 10, "categori": [10, 70], "dla": 10, "_logit": 10, "residual_stream": 10, "label": [10, 14, 15, 18, 19, 24, 38, 70, 71], "decompose_resid": [10, 70], "return_label": [10, 70], "0": [10, 12, 13, 14, 15, 16, 18, 27, 32, 36, 38, 63, 69, 70, 71, 72], "emb": [10, 17, 21, 27, 32, 69, 71], "pos_emb": [10, 15, 17, 71], "0_attn_out": 10, "proceed": 10, "space": [10, 12, 14, 69, 70, 71], "logit_attr": 10, "shape": [10, 12, 14, 18, 19, 24, 31, 32, 33, 40, 69, 70, 71], "torch": [10, 12, 13, 14, 15, 18, 24, 27, 31, 33, 37, 38, 40, 66, 68, 69, 70, 71], "10": [10, 14, 16, 69, 70, 71, 72], "7": [10, 35, 70, 71, 72], "most_important_component_idx": 10, "argmax": [10, 70], "3_attn_out": 10, "dig": [10, 70, 71, 73], "granular": 10, "get_full_resid_decomposit": 10, "stack": [10, 12, 13, 14, 40, 69, 70, 71], "equal": [10, 15, 71], "struggl": 10, "construct": [10, 12, 13], "joke": 10, "trivial": 10, "accumulated_resid": [10, 70], "footgun": [10, 37], "sourc": [10, 14, 15, 18, 35, 40, 73], "track": [10, 70], "index": [10, 12, 13, 14, 15, 16, 18, 19, 24, 38, 40, 68, 69, 70, 71], "dimens": [10, 14, 15, 18, 24, 25, 28, 37, 40, 66, 67, 69, 70, 71], "vector": [10, 11, 14, 16, 18, 40, 70, 71], "q": [10, 12, 13, 14, 18, 24, 40], "z": [10, 14, 24, 36, 40, 70, 71], "po": [10, 12, 13, 14, 18, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 40, 67, 69, 70, 71], "head_index": [10, 14, 16, 18, 19, 24, 25, 26, 30, 31, 40, 67, 70, 71], "d_head": [10, 12, 13, 14, 15, 18, 19, 24, 38, 39, 67, 69, 70, 71, 72], "pattern": [10, 12, 13, 14, 18, 19, 24, 36, 40, 71], "softmax": [10, 14, 15, 18, 19, 24, 26, 69, 71], "attn_scor": [10, 15, 18, 19], "pre": [10, 14, 15, 18, 19, 24, 26, 29, 36, 69], "query_po": [10, 18, 19, 24, 70], "key_po": [10, 18, 19, 24, 70], "d_model": [10, 12, 13, 14, 15, 18, 19, 20, 22, 23, 24, 25, 26, 27, 28, 31, 33, 34, 38, 67, 69, 70, 71, 72], "mid": [10, 70], "solu_ln": [10, 15], "layernorm": [10, 12, 13, 14, 15, 20, 25, 26, 28, 69, 70], "d_mlp": [10, 12, 13, 14, 15, 26, 38, 69, 71, 72], "resid_pr": [10, 15, 20, 31, 33, 40, 70, 71], "resid_mid": [10, 40], "resid_post": [10, 15, 70], "attn_out": [10, 14, 15, 40, 70], "mlp_out": [10, 14, 15, 40, 70], "ln": [10, 14, 15, 70, 71], "lnpre": [10, 15], "scale": [10, 14, 15, 18, 19, 24, 69, 70, 71], "sometim": [10, 35, 70], "miss": [10, 70], "becaus": [10, 11, 12, 13, 14, 15, 18, 35, 69, 70, 71, 73], "appli": [10, 14, 15, 18, 20, 33, 37, 40, 69, 70, 71], "remove_batch_dim": [10, 37, 69, 71], "batch_siz": [10, 12, 32, 35, 37, 39, 63, 70, 71], "annot": [10, 71], "layers_cov": 10, "queri": [10, 12, 13, 14, 15, 18, 19, 24, 30, 36, 40, 71], "batch_and_pos_dim": 10, "ve": [10, 14, 18, 35, 37, 70, 73], "slice": [10, 37, 69, 70], "dictionari": [10, 12, 13, 14, 15, 36, 37, 38, 69, 71], "whether": [10, 12, 14, 15, 18, 32, 35, 37, 38, 40, 63, 69, 70, 71], "int": [10, 11, 12, 13, 14, 15, 16, 18, 19, 21, 23, 24, 25, 27, 28, 30, 31, 32, 33, 35, 36, 37, 38, 39, 40, 63, 68, 69, 70, 71], "incl_mid": [10, 70], "fals": [10, 12, 13, 14, 15, 30, 35, 36, 37, 38, 39, 40, 63, 69, 70, 71], "apply_ln": [10, 70], "pos_slic": [10, 37, 70], "union": [10, 12, 13, 14, 15, 16, 18, 19, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 32, 33, 34, 36, 37, 38, 39, 40, 68, 69, 70], "tupl": [10, 11, 12, 13, 14, 18, 24, 36, 37, 40, 69, 71], "ndarrai": [10, 14, 37, 69], "mlp_input": [10, 14], "float": [10, 11, 12, 13, 14, 15, 18, 20, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 33, 34, 36, 38, 39, 40, 63, 66, 67, 69, 70, 71], "accumul": [10, 14, 70], "sub": [10, 71], "www": 10, "lesswrong": 10, "ackrb8wdpdan6v6ru": 10, "believ": [10, 35, 70], "vocabulari": [10, 15, 70, 71], "rememb": 10, "norm": [10, 11, 14, 15, 21, 25, 26, 28, 29, 38, 63, 70, 71], "decod": [10, 13, 15, 31], "therefor": [10, 14, 69], "multipli": [10, 14, 18, 19, 36, 66, 70, 71], "unembed": [10, 12, 13, 14, 70, 71], "matrix": [10, 11, 12, 13, 14, 15, 16, 18, 24, 36, 69, 70], "w_u": [10, 12, 13, 14, 70, 71], "broken": [10, 37, 69, 70, 71], "down": [10, 14, 18, 19, 24, 70, 71], "einop": [10, 70, 71], "einsum": [10, 70, 71], "panda": [10, 40], "pd": [10, 40], "devic": [10, 12, 13, 14, 15, 16, 18, 30, 35, 37, 38, 39, 63, 64, 70, 71], "answer_token": [10, 70], "to_single_token": [10, 14, 70, 71], "2975": 10, "accum_resid": 10, "last_token_accum": 10, "9": [10, 69, 70, 71, 72], "64": [10, 38, 69, 71, 72], "50257": [10, 38, 71, 72], "layers_unembed": 10, "d_vocab": [10, 12, 13, 14, 15, 16, 38, 40, 69, 71, 72], "rank": [10, 11, 14, 18, 69, 70, 71], "correct": [10, 40, 68, 69, 70, 71], "sorted_indic": 10, "argsort": 10, "dim": [10, 14, 69, 70, 71], "descend": [10, 71], "rank_answ": 10, "nonzero": 10, "as_tupl": 10, "0_pre": 10, "4442": 10, "1_pre": [10, 70], "382": 10, "2_pre": 10, "982": 10, "3_pre": 10, "1160": 10, "4_pre": 10, "408": 10, "5_pre": 10, "145": 10, "6_pre": 10, "78": 10, "7_pre": 10, "387": 10, "final_post": 10, "6": [10, 14, 38, 69, 70, 71, 72], "dtype": [10, 12, 13, 14, 15, 18, 38, 68, 70, 71], "int64": [10, 69], "exclud": [10, 36], "n_layer": [10, 12, 13, 14, 15, 38, 40, 68, 70, 71, 72], "immedi": [10, 19, 69, 70, 71], "indic": [10, 12, 13, 32, 37, 40, 69, 70, 71], "taken": [10, 14, 71], "l": [10, 12, 13, 14, 70, 71], "noth": [10, 12, 13, 14, 37, 69, 70, 71], "essenti": [10, 14, 70, 71, 73], "rather": [10, 14, 15, 20, 40, 69, 70, 71], "graph": [10, 70, 71], "apply_ln_to_stack": [10, 14, 69, 70], "residual_stack": [10, 70], "num_compon": 10, "batch_slic": 10, "batch_and_pos_dims_out": 10, "treat": [10, 14, 15, 70, 71], "factor": [10, 11, 14, 15, 70], "simul": [10, 14, 70, 71], "global": [10, 14, 18, 19, 24, 30, 37, 38, 69, 70, 71], "element": [10, 14, 18, 36, 40, 69, 71], "rmsnorm": [10, 15, 28], "unchang": [10, 12, 13, 14, 69, 70, 71], "whose": [10, 12, 13, 14, 35, 69, 70], "trail": [10, 11, 69], "assum": [10, 12, 13, 14, 15, 25, 28, 32, 37, 40, 63, 69, 70], "hook_scal": [10, 69, 70, 71], "unemb": [10, 14, 15, 17, 70, 71], "map": [10, 12, 13, 14, 15, 18, 36, 37, 70, 71], "ie": [10, 12, 13, 14, 15, 18, 38, 40, 69, 70, 71], "ln2": [10, 33, 69, 71], "ln1": [10, 15, 33, 69, 71], "ln_final": [10, 14, 70, 71], "apply_slice_to_batch_dim": 10, "compute_head_result": 10, "sum": [10, 11, 14, 15, 21, 36, 69, 70, 71], "plu": 10, "b_o": [10, 12, 13, 14, 71], "intend": [10, 15, 69], "use_attn_result": [10, 14, 15], "forget": 10, "liter": [10, 12, 13, 14, 16, 36, 37, 40], "incl_emb": 10, "decompos": 10, "incl": 10, "expand_neuron": 10, "bias": [10, 12, 13, 14, 15, 63, 70], "expand": [10, 14, 24], "get_neuron_result": 10, "neuron_slic": 10, "num_neuron": 10, "subset": [10, 15, 35, 70, 71], "specifi": [10, 12, 13, 14, 15, 24, 35, 36, 37, 68, 69, 71], "expens": [10, 11, 71], "cheap": 10, "hook_emb": [10, 69, 71], "hook_pos_emb": [10, 71], "block": [10, 14, 15, 18, 19, 20, 24, 31, 33, 40, 69, 70, 71], "hook_resid_pr": [10, 71], "incorrect_token": [10, 70], "typic": [10, 12, 14, 32, 36, 70, 71], "revers": [10, 11, 40, 69, 70, 71, 73], "dot": [10, 14, 18, 19, 69], "product": [10, 11, 12, 13, 18, 19, 71], "incorrect": [10, 14, 40, 70, 71], "arxiv": [10, 14, 15, 18, 24, 32, 35], "org": [10, 14, 15, 18, 24, 32, 35, 71, 73], "ab": [10, 11, 14, 24, 36, 70, 71], "2211": [10, 35], "00593": [10, 35], "john": [10, 70, 71], "mari": [10, 70, 71], "went": [10, 70, 71], "shop": [10, 70, 71], "gave": [10, 35, 70, 71], "bag": [10, 70], "choos": [10, 70, 71], "final_ln": 10, "residual_stack_item": 10, "dure": [10, 15, 24, 37, 39, 71, 73], "stack_activ": 10, "activation_nam": [10, 40, 70], "sublayer_typ": 10, "flexibl": 10, "given": [10, 11, 12, 13, 14, 16, 36, 37, 38, 40, 68, 69, 70, 71], "get_act_nam": [10, 69, 70, 71], "infer": [10, 14, 26, 40, 70, 71], "incl_remaind": 10, "stack_head_result": [10, 70], "axi": [10, 18, 40, 69, 70, 71], "n_head": [10, 12, 13, 14, 15, 18, 24, 38, 39, 40, 69, 70, 71, 72], "notat": [10, 70, 73], "l0h0": 10, "stack_neuron_result": 10, "l0n0": 10, "super": [10, 14, 71], "short": [10, 69, 70, 71, 73], "mostli": [10, 70, 71], "finish": [10, 14, 69, 70, 71], "oper": [10, 70, 71], "slower": 10, "unless": [10, 14, 15, 35, 71], "deprec": 10, "toggle_autodiff": 10, "toggl": [10, 14], "autodiff": [10, 71], "set_grad_en": [10, 70, 71], "danger": 10, "off": [10, 14, 35, 69, 70, 71], "realis": [10, 30, 70], "downstream": 10, "delet": [10, 69, 70], "stick": [10, 70], "mess": [10, 14, 69, 71], "inference_mod": 10, "decor": 10, "similar": [10, 12, 13, 14, 20, 24, 36, 70, 71], "requires_grad": 10, "eigenvalu": 11, "ldim": [11, 71], "mdim": [11, 71], "rdim": [11, 71], "leading_dim": [11, 69], "ba": 11, "vh": [11, 14], "collapse_l": 11, "collaps": [11, 70, 71], "left": [11, 14, 18, 69, 70, 71, 73], "orthogon": [11, 14], "self": [11, 12, 13, 14, 18, 69, 71], "collapse_r": 11, "analog": [11, 70, 71], "apart": [11, 69, 70, 71], "zero": [11, 14, 18, 36, 69, 70, 71], "bav": 11, "kv": 11, "abav": 11, "kav": 11, "av": 11, "eigenvector": [11, 71], "get_corn": [11, 69, 70], "make_even": 11, "sqrt": [11, 14, 15, 69], "diag": 11, "equival": [11, 14, 18, 70, 71], "factoris": [11, 14, 18, 71], "row": [11, 14, 40], "col": 11, "ndim": 11, "frobeniu": [11, 71], "squar": [11, 28, 29, 69, 71], "m": [11, 18, 66, 69, 70, 71], "st": 11, "transpos": [11, 69], "obviou": [11, 14, 70], "unsqueez": [11, 69], "hook": [12, 13, 14, 15, 19, 20, 21, 22, 23, 25, 26, 27, 28, 29, 32, 33, 34, 37, 70], "encod": [12, 13, 18, 31, 69, 71], "contain": [12, 13, 14, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 32, 33, 34, 35, 36, 37, 38, 39, 69, 70, 71], "bert": [12, 20, 21, 22, 32, 38, 42, 71, 72], "move_to_devic": [12, 13, 14], "kwarg": [12, 13, 14, 38, 69, 70, 71], "hookedrootmodul": [12, 13, 14, 37, 71], "hookpoint": [12, 13, 14, 37, 70, 71], "inherit": [12, 13, 71], "mvp": 12, "mask": [12, 13, 14, 18, 22, 27, 31, 33, 69, 70], "mlm": [12, 22], "next": [12, 14, 69, 70, 71], "sentenc": [12, 13, 14, 22, 32, 36, 70, 71], "nsp": 12, "dropout": [12, 13], "inconsist": [12, 13, 16], "fine": [12, 13, 71], "fold": [12, 13, 14, 18, 26, 38, 70], "ov": [12, 13, 14, 16, 18, 70, 71], "o": [12, 13, 66, 71], "qk": [12, 13, 14, 18, 70], "w_e": [12, 13, 14, 71], "conveni": [12, 13, 14, 15, 37, 69, 71], "w_e_po": [12, 14], "n_ctx": [12, 14, 15, 18, 38, 71, 72], "concaten": [12, 14, 69, 70, 71], "w_po": [12, 13, 14, 71], "overcomplet": [12, 14], "w_k": [12, 13, 14, 15, 18, 24, 71], "w_o": [12, 13, 14, 18, 19, 70, 71], "w_q": [12, 13, 14, 18, 24, 71], "w_v": [12, 13, 14, 18, 24, 71], "w_in": [12, 13, 14, 16, 71], "w_out": [12, 13, 14, 16, 71], "all_head_label": [12, 13, 14], "format": [12, 13, 14, 37, 70, 71], "h": [12, 13, 14, 70, 71], "b_k": [12, 13, 14, 18, 24, 71], "b_q": [12, 13, 14, 71], "b_u": [12, 13, 14, 70, 71], "bia": [12, 13, 14, 15, 18, 28, 29, 30, 66, 70, 71], "b_v": [12, 13, 14, 18, 24, 71], "b_in": [12, 13, 14, 71], "b_out": [12, 13, 14, 71], "buffer": [12, 13, 14], "modifi": [12, 13, 14], "cuda": [12, 13, 14, 15, 35, 38], "associ": [12, 13, 14, 37], "optim": [12, 13, 14, 63, 70], "return_typ": [12, 13, 14, 37, 70, 71], "token_type_id": [12, 21, 32], "one_zero_attention_mask": [12, 13], "binari": [12, 13, 32], "id": [12, 14, 15, 32], "belong": [12, 32], "cl": [12, 32, 71], "sep": [12, 32], "sequence_length": [12, 32, 36, 69], "attend": [12, 13, 15, 18, 19, 24, 69, 70, 71], "ignor": [12, 13, 14, 15, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 69, 71], "primarili": [12, 13], "pad": [12, 13, 14, 18, 27, 31, 33, 69, 70, 71], "shorter": [12, 13, 14, 71], "classmethod": [12, 13, 14, 15, 39, 69], "model_nam": [12, 13, 14, 15, 38, 71], "checkpoint_index": [12, 13, 14, 15, 38, 71], "checkpoint_valu": [12, 13, 14, 15, 38, 71], "hf_model": [12, 13, 14], "float32": [12, 13, 14, 15, 18, 38, 70], "from_pretrained_kwarg": [12, 13, 14], "bertformaskedlm": [12, 13], "unlik": [12, 13, 14, 40, 71], "mp": [12, 13, 14], "model_arg": [12, 13, 14, 37], "return_cache_object": [12, 13, 14], "otherwis": [12, 13, 14, 35, 36, 69], "device_or_dtyp": [12, 13, 14, 68], "print_detail": [12, 13, 14, 68, 69], "cast": [12, 13, 14], "non_block": [12, 13, 14], "memory_format": [12, 13, 14], "channels_last": [12, 13, 14], "Its": [12, 13, 14], "complex": [12, 13, 14, 15, 70, 71], "integr": [12, 13, 14, 38], "tri": [12, 13, 14, 70, 71, 73], "asynchron": [12, 13, 14], "respect": [12, 13, 14, 37, 69, 71], "host": [12, 13, 14, 38], "pin": [12, 13, 14], "desir": [12, 13, 14], "4d": [12, 13, 14], "keyword": [12, 13, 14, 37, 71], "argument": [12, 13, 14, 15, 37, 38, 69, 71], "xdoctest": [12, 13, 14], "ignore_w": [12, 13, 14], "non": [12, 13, 14, 15, 18, 35, 69, 70, 71], "determinist": [12, 13, 14, 69, 70], "nn": [12, 13, 14, 37, 71], "1913": [12, 13, 14], "3420": [12, 13, 14], "5113": [12, 13, 14], "2325": [12, 13, 14], "doubl": [12, 13, 14], "in_featur": [12, 13, 14], "out_featur": [12, 13, 14], "float64": [12, 13, 14], "env": [12, 13, 14], "torch_doctest_cuda1": [12, 13, 14], "gpu1": [12, 13, 14], "1914": [12, 13, 14], "5112": [12, 13, 14], "2324": [12, 13, 14], "float16": [12, 13, 14], "cdoubl": [12, 13, 14], "3741": [12, 13, 14], "j": [12, 13, 14, 15, 18, 38, 70, 71, 72], "2382": [12, 13, 14], "5593": [12, 13, 14], "4443": [12, 13, 14], "complex128": [12, 13, 14], "6122": [12, 13, 14], "1150": [12, 13, 14], "encoderdecod": [13, 15], "t5": [13, 15, 30, 31, 38, 42, 72], "decoder_input": 13, "decoder_po": 13, "usual": [13, 31], "fairli": [14, 70, 71], "extract": [14, 71], "harder": [14, 40, 70], "aim": [14, 70, 73], "simplifi": [14, 70, 71], "attach": [14, 71], "within": [14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 36, 37, 40, 69, 70, 71], "inspect": [14, 70], "alter": 14, "facilit": 14, "deeper": 14, "pretrainedtokenizerbas": 14, "default_padding_sid": 14, "50": [14, 63, 71], "initialis": [14, 15], "although": [14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "instanti": [14, 15, 71], "__init__": [14, 18, 19, 24, 25, 26, 28, 29, 37, 69, 71], "test_prompt": [14, 69, 70, 71], "w_gate": 14, "tokenizer_nam": [14, 15], "explicitli": [14, 15, 18, 35, 38, 40, 71], "n_devic": [14, 15, 38, 68], "greater": [14, 36], "multipl": [14, 36, 68, 69, 71], "accumulated_bia": 14, "include_mlp_bias": 14, "all_composition_scor": [14, 69], "score": [14, 15, 18, 19, 24, 36, 40, 70], "l1": 14, "h1": 14, "l2": 14, "h2": [14, 70], "upper": [14, 18], "triangular": [14, 36, 69, 71], "third": [14, 71], "pub": [14, 69], "2021": 14, "framework": [14, 18, 70, 71], "html": [14, 69, 70], "20abov": 14, "20diagram": 14, "20show": 14, "20q": 14, "2d": [14, 66, 69], "2c": 14, "20k": [14, 69], "20and": 14, "20v": 14, "2dcomposit": 14, "metric": [14, 36, 40, 70, 71], "center_unemb": [14, 70], "state_dict": 14, "center": [14, 15, 26, 28, 29, 70, 71], "subtract": [14, 36, 70], "translat": [14, 70, 71], "invari": 14, "prob": [14, 69, 70, 71], "slightli": [14, 69, 70], "misl": 14, "center_writing_weight": [14, 70, 71], "fold_layer_norm": [14, 38], "check_hooks_to_add": [14, 37], "hook_point_nam": [14, 37], "dir": [14, 37], "fwd": [14, 37], "is_perman": [14, 37], "prepend": [14, 15, 35, 37, 38, 69, 71], "overrid": [14, 15, 37, 38, 69], "fold_bias": 14, "center_weight": 14, "rm": [14, 15, 28, 29], "neighbour": 14, "further_com": [14, 15], "md": [14, 15], "fold_value_bias": 14, "alwai": [14, 15, 40, 70, 71], "constant": [14, 15, 18, 70, 71], "doesn": [14, 35, 69, 70, 71], "formal": 14, "b_o_new": 14, "b_o_origin": 14, "sum_head": 14, "b_v_head": 14, "w_o_head": 14, "loss_per_token": 14, "prepend_bo": [14, 15, 35, 38, 69, 70], "use_default_valu": 14, "padding_sid": [14, 69, 70], "start_at_lay": 14, "shortformer_pos_emb": [14, 18, 33], "attention_mask": [14, 18, 27, 31, 33, 39, 69], "stop_at_lay": [14, 71], "past_kv_cach": [14, 27], "hookedtransformerkeyvaluecach": [14, 31, 33, 39], "flag": [14, 15, 35, 37, 40, 69, 70, 71], "entropi": [14, 69, 70, 71], "per": [14, 40, 70, 71], "averag": [14, 35, 70, 71], "scalar": [14, 18, 37, 71], "default_prepend_bo": [14, 15, 35, 38, 69, 71], "bo": [14, 15, 36, 38, 69, 70, 71], "impli": 14, "usag": [14, 70], "accordingli": [14, 15, 18, 38, 70, 71], "lose": [14, 15, 38], "empir": [14, 15, 38, 40, 71], "inclus": 14, "neg": [14, 69, 70, 71], "shortform": [14, 15, 18, 33, 38], "positional_embedding_typ": [14, 15, 18], "stop": [14, 71], "exclus": [14, 69], "etc": [14, 15, 40, 70, 71, 73], "frozen": [14, 39], "pai": [14, 18, 70], "okai": 14, "twice": [14, 35, 70, 71], "accident": [14, 37], "fold_ln": [14, 38, 70, 71], "refactor_factored_attn_matric": [14, 70], "automodelforcausallm": 14, "first_n_lay": [14, 38], "autoregress": [14, 63], "neo": [14, 18, 19, 24, 38, 42, 71, 72], "gptj": [14, 38, 42], "opt": [14, 38, 42, 71, 72], "solu": [14, 15, 38, 69, 71, 72], "checkpoint": [14, 15, 38, 63], "neelnanda": [14, 38], "stanford": [14, 15, 18, 19, 24, 38, 71, 72], "crfm": [14, 38, 71], "load_and_process_state_dict": 14, "alia": [14, 37, 38, 69, 71], "subsequ": [14, 38, 70, 71], "regular": [14, 18, 24], "batchnorm": [14, 70, 71], "mathemat": [14, 18, 70, 71], "w_": 14, "b_": 14, "w": [14, 67], "layernormpr": [14, 26, 29], "eff": 14, "ext": 14, "wise": [14, 36], "computation": [14, 71], "wish": 14, "defin": [14, 18, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 69, 70, 71], "x_1": [14, 71], "x_0": [14, 71], "x_2": [14, 71], "frac": [14, 71], "x_3": 14, "cdot": 14, "x_4": 14, "preced": [14, 69, 70, 71], "never": [14, 71], "w_write": 14, "keepdim": 14, "fed": [14, 36], "1000": [14, 35, 69, 71], "recreat": 14, "onto": [14, 38, 70], "By": [14, 35, 37, 38, 40, 69, 70, 71], "mix": [14, 69, 70, 71], "linearli": 14, "technic": [14, 70, 71], "deriv": [14, 71], "broadcast_b_v": 14, "broadcast": 14, "And": [14, 40, 70, 71], "destination_posit": [14, 71], "source_posit": [14, 71], "source_": 14, "destin": [14, 15, 40, 71], "behavior": [14, 15, 38, 70], "resolut": [14, 38], "cfg_dict": [14, 38], "cache_dir": [14, 69], "torch_dtyp": 14, "bfloat16": 14, "boolean": [14, 37, 40, 69, 70, 71], "max_new_token": [14, 71], "stop_at_eo": 14, "eos_token_id": [14, 69], "do_sampl": 14, "top_k": [14, 69, 70, 71], "top_p": [14, 69], "temperatur": [14, 69, 71], "freq_penalti": [14, 69], "use_past_kv_cach": 14, "verbos": 14, "pos_plus_new_token": 14, "eos_token": 14, "reach": [14, 71], "avoid": [14, 15, 39, 69, 70, 71], "fiddl": 14, "rag": 14, "eot": 14, "throw": 14, "enter": [14, 70, 71, 73], "messi": [14, 71], "maximum": [14, 15, 18, 63, 71], "stable_lm": 14, "distribut": [14, 68, 69, 70, 71], "greedi": [14, 69], "search": [14, 36, 70, 71], "max": [14, 70], "mass": 14, "random": [14, 15, 35, 63, 70, 71], "temp": [14, 69], "inf": 14, "uniform": [14, 69], "frequenc": [14, 15, 69, 70], "penalti": [14, 69], "penalis": 14, "speed": [14, 70], "applic": [14, 15, 69], "whatev": [14, 70], "tqdm": [14, 71], "get_token_posit": [14, 70, 71], "single_token": [14, 71], "present": 14, "gotcha": [14, 16, 70], "Be": 14, "care": [14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 70, 71], "weird": [14, 15, 70, 71], "carefulli": [14, 70], "correspond": [14, 24, 36, 40, 69, 70, 71], "dummi": [14, 37, 71], "init_weight": [14, 15], "empti": [14, 37], "bulk": 14, "seed": [14, 15, 63, 71], "determin": [14, 18, 40, 68, 69, 70, 71], "NOT": [14, 37, 69, 71], "scheme": 14, "tell": [14, 35, 70, 71], "round": [14, 35, 70, 71], "18182": 14, "fan_in": [14, 69], "tha": 14, "kaim": [14, 69], "despit": [14, 71], "xavier": [14, 69], "fan_out": 14, "transformerencod": 14, "exact": 14, "72253": 14, "mup": [14, 15], "haven": 14, "2203": 14, "03466": 14, "input_to_emb": 14, "special": [14, 71], "redwood": [14, 70, 71], "load_sample_training_dataset": 14, "dataset": [14, 35, 63, 69, 71], "10k": [14, 35, 69], "get_dataset": [14, 69], "appropri": [14, 71], "info": [14, 15, 40, 69, 71], "download": [14, 69, 71], "locat": [14, 40, 70, 71], "pt": 14, "openwebtext": [14, 35, 69], "karma": [14, 35], "reddit": [14, 35], "pile": [14, 35, 38, 69, 71, 72], "imperfectli": 14, "suppli": 14, "valid": [14, 35, 70], "loss_fn": [14, 71], "per_token": [14, 69, 71], "lm_cross_entropy_loss": [14, 69], "move_model_modules_to_devic": 14, "process_weights_": 14, "cleaner": 14, "experiment": 14, "argu": [14, 71], "somewhat": [14, 70, 71], "w_qk": [14, 18, 71], "w_ov": [14, 18, 71], "hopefulli": [14, 73], "column": [14, 40, 69], "rotat": [14, 15, 18, 71], "nth": 14, "formula": 14, "r": 14, "refactor": 14, "diagon": [14, 70, 71], "asymmetri": 14, "fiddli": 14, "preserv": [14, 70, 71], "too": [14, 40, 70], "bilinear": [14, 71], "dimension": [14, 15], "coordin": 14, "sample_datapoint": 14, "implicitli": [14, 40, 71], "hasn": 14, "manual": [14, 69, 71], "replac": [14, 15, 40, 70, 71, 73], "choic": [14, 70], "truncat": [14, 35, 69, 71], "set_token": [14, 15], "pretrainedtoken": 14, "set_ungroup_grouped_query_attent": 14, "ungroup_grouped_query_attent": [14, 15], "ungroup": [14, 15], "gqa": 14, "set_use_attn_in": 14, "use_attn_in": [14, 15], "set_use_attn_result": 14, "easili": [14, 69, 70, 71], "burn": 14, "set_use_hook_mlp_in": 14, "use_hook_mlp_in": [14, 15], "set_use_split_qkv_input": 14, "use_split_qkv_input": [14, 15], "to_single_str_token": 14, "int_token": 14, "uncertain": 14, "to_token": [14, 69, 70, 71], "to_str_token": [14, 16, 70, 71], "weirdli": [14, 70, 71], "gotcha2": 14, "letter": [14, 71], "capit": [14, 70, 71], "shoot": [14, 71], "gotcha3": 14, "exce": 14, "str_token": [14, 70], "to_str": [14, 70, 71], "numpi": [14, 15, 69, 70], "arrai": [14, 16, 69], "long": [14, 71], "window": [14, 15, 69], "tokens_to_residual_direct": [14, 70], "mislead": [14, 70], "integ": [14, 69, 70, 71], "residual_direct": 14, "namedtupl": 14, "dataclass": [15, 37], "act_fn": [15, 72], "ep": 15, "1e": [15, 38], "05": [15, 38], "use_attn_scal": 15, "attn_scal": 15, "use_local_attn": 15, "original_architectur": 15, "from_checkpoint": 15, "checkpoint_label_typ": [15, 71], "window_s": [15, 18, 19, 24], "attn_typ": [15, 18, 19, 24, 30], "init_mod": 15, "normalization_typ": 15, "attention_dir": 15, "attn_onli": [15, 72], "initializer_rang": 15, "scale_attn_by_inverse_layer_idx": 15, "final_rm": 15, "d_vocab_out": [15, 34], "parallel_attn_mlp": 15, "rotary_dim": [15, 18], "n_param": [15, 72], "use_hook_token": 15, "gated_mlp": 15, "tokenizer_prepends_bo": 15, "n_key_value_head": [15, 24, 72], "post_embedding_ln": 15, "rotary_bas": 15, "10000": [15, 18, 71], "trust_remote_cod": 15, "rotary_adjacent_pair": 15, "load_in_4bit": 15, "num_expert": 15, "experts_per_token": 15, "relative_attention_max_dist": 15, "relative_attention_num_bucket": 15, "decoder_start_token_id": 15, "tie_word_embed": 15, "use_normalization_before_and_aft": 15, "attn_scores_soft_cap": 15, "output_logits_soft_cap": 15, "use_ntk_by_parts_rop": 15, "ntk_by_parts_low_freq_factor": 15, "ntk_by_parts_high_freq_factor": 15, "ntk_by_parts_factor": 15, "8": [15, 18, 35, 36, 70, 71, 72], "AND": 15, "feedforward": 15, "network": [15, 70, 71], "vocab": 15, "lowercas": 15, "relu": [15, 69, 72], "gelu": [15, 38, 71, 72], "silu": [15, 72], "gelu_new": [15, 69], "gelu_fast": [15, 69], "epsilon": 15, "5": [15, 18, 35, 36, 38, 40, 69, 70, 71, 72], "THEN": 15, "intens": 15, "divid": [15, 36, 69, 70], "distanc": [15, 18, 70], "xavier_uniform": 15, "xavier_norm": 15, "kaiming_uniform": 15, "kaiming_norm": 15, "rmspre": 15, "pipelin": 15, "parallel": [15, 69, 70], "aka": 15, "unidirect": 15, "bidirect": [15, 71], "gain": [15, 69], "layer_id": [15, 18, 19, 24, 30], "numer": [15, 16, 18, 19, 24, 71], "stabil": [15, 18, 19, 24, 71], "fp16": 15, "rotari": [15, 18], "blog": [15, 18], "eleuth": [15, 18, 69, 71], "res_stream": 15, "sinusoid": 15, "dumb": 15, "mainli": 15, "curs": 15, "init": 15, "law": 15, "pdf": [15, 18, 32, 35], "2001": 15, "08361": 15, "Will": [15, 40], "interven": [15, 37, 40, 70], "add_bos_token": [15, 69], "bitsandbyt": 15, "moe": 15, "bucket": 15, "tie": 15, "word": [15, 69, 70, 71], "gemma": [15, 38, 42, 72], "softcap": 15, "soft_cap": 15, "tanh": 15, "squash": 15, "interv": [15, 36], "ntk": 15, "adjust": [15, 70, 71], "interpol": 15, "2309": 15, "00071": 15, "threshold": 15, "high": [15, 69, 70, 71], "rate": [15, 63, 71], "strategi": [15, 71], "from_dict": 15, "config_dict": 15, "is_layer_norm_activ": 15, "set_seed_everywher": 15, "to_dict": 15, "unwrap": [15, 69], "duplic": [15, 36, 70, 71], "get_singular_vector": 16, "vector_typ": 16, "layer_index": [16, 70], "num_vector": 16, "plot": [16, 71], "pysvelt": [16, 71], "instabl": 16, "d": [16, 35, 36, 38, 70, 72], "medium": [16, 38, 72], "svd_interpret": 16, "22": [16, 35, 69, 70, 71], "all_token": 16, "np": [16, 69, 70], "def": [16, 70, 71], "plot_matrix": 16, "filter": [16, 37, 38, 69, 71], "topk": [16, 70], "topktabl": 16, "obj_typ": 16, "abstract_attent": 17, "bert_block": 17, "bert_emb": 17, "bert_mlm_head": 17, "grouped_query_attent": 17, "layer_norm": [17, 70], "layer_norm_pr": 17, "rms_norm": 17, "rms_norm_pr": 17, "t5_attent": 17, "t5_block": 17, "token_typed_emb": 17, "transformer_block": 17, "abstractattent": [18, 19, 24, 30], "abc": [18, 71], "pure": 18, "glossari": 18, "sorri": 18, "underli": [18, 40, 70, 71], "destination_residu": 18, "destination_po": 18, "source_po": [18, 71], "abstract": [18, 70, 71], "groupedqueryattent": [18, 24], "enforc": 18, "child": 18, "better_abc": 18, "abstract_attribut": 18, "stackoverflow": 18, "question": [18, 70, 71], "23831510": 18, "256": [18, 19, 24, 71, 72], "alibi": 18, "apply_causal_mask": 18, "pos_plus_past_kv_pos_offset": 18, "past_kv_pos_offset": [18, 27, 69], "offset_po": [18, 27, 33, 69], "apply_rotari": 18, "calculate_attention_scor": [18, 24], "calculate_qkv_matric": [18, 24], "query_input": [18, 24], "key_input": [18, 24], "kv_po": [18, 30, 31], "value_input": [18, 24], "calculate_sin_cos_rotari": 18, "sine": 18, "cosin": 18, "wave": 18, "inexplic": 18, "adjac": [18, 70], "neox": [18, 38, 42, 71, 72], "clue": [18, 70], "resolv": 18, "calculate_z_scor": [18, 24], "static": [18, 35], "create_alibi_bia": 18, "head_idx": 18, "2108": 18, "12409": 18, "broad": [18, 70], "proport": [18, 69], "distant": 18, "0000": [18, 70], "0625": 18, "1250": 18, "1875": 18, "0039": 18, "0078": 18, "0117": 18, "create_alibi_multipli": 18, "geometr": 18, "ratio": [18, 69, 70, 71], "16": [18, 69, 70, 71, 72], "5000": 18, "2500": [18, 70], "0312": 18, "0156": 18, "7071": 18, "3536": 18, "1768": 18, "0884": 18, "0442": 18, "0221": 18, "0110": 18, "0055": 18, "create_alibi_slop": 18, "slope": 18, "triangl": 18, "lower": [18, 35, 36, 69, 70, 71], "bottom": [18, 71], "corner": 18, "kv_head_index": [18, 24], "past_kv_cache_entri": [18, 31, 33], "hookedtransformerkeyvaluecacheentri": [18, 31, 33, 39], "additive_attention_mask": [18, 20, 31], "position_bia": [18, 30, 31], "irrelev": [18, 70, 71], "past": [18, 39, 70], "rotate_every_two": 18, "x0": 18, "x1": 18, "param": [19, 63, 69, 71], "convent": [19, 69, 70, 71], "mistal": [19, 24], "bertblock": 20, "transformerblock": [20, 33], "except": [20, 70, 71], "overridden": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 69], "subclass": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "recip": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "afterward": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "former": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "regist": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "latter": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 71], "silent": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "bertemb": 21, "input_id": 21, "bertmlmhead": 22, "purpos": [22, 35, 70, 71], "resid": 22, "2305": 24, "13245": 24, "hood": 24, "_w_k": 24, "_w_v": 24, "getter": 24, "similarli": 24, "kept": 24, "repeat_interleav": 24, "unexpand": 24, "expan": 24, "n_query_head": 24, "gpa": 24, "normalis": [26, 70], "posemb": 27, "root": [28, 29, 71], "rmsnormpr": 29, "t5attent": [30, 31], "has_relative_attention_bia": 30, "expect": [30, 35, 36, 70, 71], "precomput": 30, "feed": [30, 35, 69, 71], "positional_bia": 30, "compute_relative_attention_bia": 30, "query_len": 30, "key_len": 30, "query_length": 30, "key_length": 30, "bin": 30, "t5block": 31, "block_index": [31, 33], "is_decod": 31, "t5layernorm": 31, "inst": 31, "encoder_additive_attention_mask": 31, "encoder_po": 31, "encoder_hidden_st": 31, "_description_": 31, "_type_": [31, 37], "tokentypeemb": 32, "1810": 32, "04805": 32, "apply_mlp": 33, "normalized_resid": 33, "central": [33, 70], "positional_embeddings_typ": 33, "canbeusedasmlp": 33, "evalu": [35, 37, 70, 71], "rough": [35, 71], "cheapli": 35, "roughli": [35, 70, 71], "baselin": 35, "ioidataset": 35, "noun": 35, "num_sampl": 35, "symmetr": 35, "ioi_ev": 35, "476": 35, "met": 35, "alic": 35, "bob": 35, "charli": 35, "ball": [35, 70], "book": 35, "397": 35, "get_default_nam": 35, "get_default_noun": 35, "get_default_templ": 35, "get_sampl": 35, "evaluate_on_dataset": 35, "data_load": 35, "induction_loss": [35, 71], "subseq_len": 35, "384": [35, 71], "io": [35, 36, 70, 71], "accuraci": [35, 36, 69], "make_code_data_load": 35, "codeparrot": [35, 69], "dump": 35, "presum": [35, 70], "natur": [35, 70, 71], "make_owt_data_load": 35, "corpu": [35, 69], "make_pile_data_load": 35, "eleutherai": [35, 38], "english": [35, 71, 73], "academ": 35, "internet": [35, 71], "make_wiki_data_load": 35, "wikitext": 35, "wikipedia": [35, 69, 71], "articl": [35, 69, 70, 71], "bother": 35, "quarantin": 35, "nowadai": 35, "leakag": 35, "though": [35, 69, 70, 71], "sanity_check": 35, "paragraph": [35, 71], "zoom": [35, 40, 70], "quick": [35, 36, 71], "saniti": [35, 70], "ok": [35, 70, 71], "gone": [35, 70, 71], "wrong": [35, 37, 70], "compute_head_attention_similarity_scor": 36, "attention_pattern": [36, 71], "detection_pattern": 36, "exclude_bo": 36, "exclude_current_token": 36, "error_measur": 36, "mul": 36, "exclude_bcurrent_token": 36, "detect_head": 36, "seq": [36, 69], "previous_token_head": 36, "duplicate_token_head": 36, "induction_head": 36, "headnam": 36, "straightforward": [36, 70], "fraction": 36, "alloc": 36, "prohibit": [36, 71], "cours": [36, 70], "raw": [36, 70], "perfect": [36, 70], "examin": 36, "switch": 36, "advantag": 36, "closer": 36, "head_nam": 36, "ntensor": 36, "ioi": [36, 70, 71], "spacifi": 36, "analyz": 36, "paid": [36, 70, 71], "get_duplicate_token_head_detection_pattern": 36, "dynalist": 36, "n2zwtnoyhru1s4vnfsaq519j": 36, "2ukvedzonghl5uhugvhroxeo": 36, "get_induction_head_detection_pattern": 36, "_tfvup5csv5orithmqwj0gsi": 36, "get_previous_token_head_detection_pattern": 36, "0o5vohe9xezn8ertywkh7ioc": 36, "get_supported_head": 36, "hookfunct": 37, "_hookfunctionprotocol": 37, "inspir": [37, 71, 73], "garcon": [37, 71, 73], "ident": [37, 69, 70, 71], "wrap": [37, 71], "add_hook": [37, 70], "bwd": 37, "fn": 37, "hook_nam": 37, "add_perma_hook": [37, 71], "clear_context": 37, "remove_hook": 37, "including_perman": 37, "interfac": [37, 71, 73], "nice": [37, 70], "run_with_hook": [37, 70, 71], "temporari": [37, 69, 71], "debug": [37, 38, 63], "intent": 37, "reset_hook": [37, 71], "goe": [37, 70, 71], "reset_hooks_end": [37, 70], "add_caching_hook": 37, "names_filt": [37, 70, 71], "callabl": [37, 40], "incl_bwd": 37, "namesfilt": 37, "lambda": [37, 70, 71], "cache_som": 37, "check_and_add_hook": 37, "get_caching_hook": 37, "fwd_hook": [37, 70, 71], "bwd_hook": 37, "hook_dict": 37, "exit": [37, 69], "clear": [37, 71], "reset": 37, "my_hook": 37, "hooked_loss": 37, "mod_dict": 37, "remove_all_hook_fn": 37, "model_kwarg": 37, "degrad": 37, "lenshandl": 37, "removablehandl": 37, "context_level": 37, "hold": 37, "perman": 37, "hug": 38, "face": 38, "hub": [38, 69], "768": [38, 70, 71, 72], "layer_norm_ep": 38, "init_rang": 38, "02": 38, "1024": [38, 69, 71, 72], "3072": [38, 71, 72], "12": [38, 70, 71, 72], "model_alias": 38, "01": 38, "yi": [38, 72], "34b": [38, 72], "6b": [38, 71, 72], "arthurconmi": 38, "redwood_attn_2l": [38, 72], "baidicoot": 38, "3b": [38, 71, 72], "125m": [38, 71, 72], "20b": [38, 71, 72], "pythia": [38, 72], "4b": [38, 72], "dedup": [38, 72], "12b": [38, 72], "13b": [38, 71, 72], "14m": [38, 72], "160m": [38, 72], "seed1": [38, 72], "seed2": [38, 72], "seed3": [38, 72], "1b": [38, 72], "800m": 38, "8b": [38, 71, 72], "31m": [38, 72], "410m": [38, 72], "350m": 38, "9b": [38, 72], "70m": [38, 72], "19m": [38, 72], "2l512w": 38, "lr": [38, 63], "attn_only_1l512w_c4_cod": 38, "c4": [38, 69, 71], "attn_only_2l512w_c4_cod": 38, "attn_only_3l512w_c4_cod": 38, "attn_only_4l512w_c4_cod": 38, "gelu_1l512w_c4_cod": 38, "gelu_2l512w_c4_cod": 38, "gelu_3l512w_c4_cod": 38, "gelu_4l512w_c4_cod": 38, "solu_10l1280w_c4_cod": 38, "10l": [38, 71, 72], "solu_10l_v22_old": 38, "solu_12l1536w_c4_cod": 38, "12l": [38, 71, 72], "solu_12l_v23_old": 38, "solu_1l512w_c4_cod": 38, "solu_1l512w_wiki_finetun": 38, "wiki": [38, 69, 70, 71, 72], "finetun": 38, "solu_1l_v9_old": 38, "solu_2l512w_c4_cod": 38, "solu_2l_v10_old": 38, "solu_3l512w_c4_cod": 38, "solu_4l512w_c4_cod": 38, "solu_4l512w_wiki_finetun": 38, "solu_4l_v11_old": 38, "solu_6l768w_c4_cod": 38, "6l": [38, 71, 72], "solu_6l_v13_old": 38, "solu_8l1024w_c4_cod": 38, "8l": [38, 71, 72], "solu_8l_v21_old": 38, "qwen": [38, 42, 61, 72], "14b": [38, 72], "1_8b": 38, "qwen1": [38, 72], "5b": [38, 71, 72], "forev": 38, "mgpt": [38, 72], "bigcod": 38, "santacod": [38, 72], "bigscienc": 38, "1b1": [38, 72], "1b7": [38, 72], "560m": [38, 72], "7b1": [38, 72], "codellama": [38, 72], "hf": 38, "codellamallama": [38, 72], "distilgpt2": [38, 71], "distillgpt2": [38, 72], "distil": [38, 71], "facebook": 38, "xxl": 38, "30b": [38, 71, 72], "xxxl": 38, "xl": [38, 71, 72], "66b": [38, 71, 72], "xxxxl": 38, "27b": [38, 72], "2b": [38, 72], "65b": [38, 72], "70b": [38, 72], "microsoft": 38, "phi": [38, 42, 59, 72], "mini": 38, "4k": 38, "1_5": [38, 72], "nemo": [38, 72], "2407": [38, 72], "roneneldan": 38, "tinystori": 38, "1layer": 38, "21m": [38, 72], "28m": [38, 72], "2layer": 38, "33m": [38, 72], "3m": [38, 72], "8m": [38, 72], "instuct": 38, "stabilityai": 38, "stablelm": [38, 71, 72], "alpha": [38, 72], "x21": 38, "arwen": 38, "battlestar": 38, "x49": 38, "beren": 38, "caprica": 38, "x81": 38, "celebrimbor": 38, "darkmatt": 38, "x343": 38, "durin": 38, "eowyn": 38, "x777": 38, "expans": 38, "alias": 38, "non_hf_hosted_model_nam": 38, "qwen2": [38, 42, 72], "32b": [38, 72], "72b": [38, 72], "qwq": [38, 72], "preview": [38, 72], "get_checkpoint_label": [38, 71], "label_typ": 38, "get_num_params_of_pretrain": 38, "suffici": [38, 70], "get_pretrained_model_config": 38, "hf_cfg": 38, "automodel": 38, "autoconfig": 38, "infrastructur": [38, 70, 71, 73], "ourselv": [39, 69, 71, 73], "previous_attention_mask": 39, "pos_so_far": 39, "append": [39, 70, 71], "prefix": 39, "append_attention_mask": 39, "new_token": 39, "freez": 39, "init_cach": 39, "unfreez": 39, "past_kei": 39, "jaxtyp": [39, 70, 71], "past_valu": 39, "new_kei": 39, "new_valu": 39, "init_cache_entri": 39, "structur": [40, 71], "generic_activation_patch": 40, "specialis": [40, 70], "introduc": [40, 70], "rome": [40, 70, 71], "baulab": 40, "corrupt": [40, 70, 71], "iter": [40, 69, 70, 71], "localis": [40, 70, 71], "__from__": 40, "__to": 40, "__the": 40, "confid": [40, 70, 71], "intuit": [40, 70, 71], "diffus": [40, 70], "spread": [40, 70], "connect": [40, 70], "ultim": [40, 70], "tend": [40, 71], "extrem": [40, 70, 71, 73], "eiffel": 40, "tower": 40, "pari": 40, "factual": [40, 70], "recal": [40, 70], "colosseum": 40, "corrupted_token": [40, 70, 71], "clean_cach": [40, 70, 71], "patching_metr": 40, "patch_sett": 40, "index_axis_nam": 40, "src_po": [40, 70], "dest_po": [40, 70, 71], "index_df": 40, "datafram": 40, "return_index_df": 40, "counterfactu": [40, 70, 71], "Then": 40, "index_to_act_nam": 40, "recov": [40, 70, 71], "diff": [40, 70], "corrupted_activ": 40, "chunk": 40, "fill": 40, "flatten": [40, 70, 71], "patched_output": 40, "get_act_patch_attn_head_all_pos_everi": 40, "patch_typ": 40, "get_act_patch_attn_head_by_pos_everi": 40, "get_act_patch_attn_head_k_all_po": 40, "corruptedactiv": 40, "patchedactiv": 40, "layer_head_vector_patch_sett": 40, "axisnam": 40, "get_act_patch_attn_head_k_by_po": 40, "layer_pos_head_vector_patch_sett": 40, "get_act_patch_attn_head_out_all_po": 40, "get_act_patch_attn_head_out_by_po": 40, "get_act_patch_attn_head_pattern_all_po": 40, "layer_head_pattern_patch_sett": 40, "get_act_patch_attn_head_pattern_by_po": 40, "layer_head_pos_pattern_patch_sett": 40, "get_act_patch_attn_head_pattern_dest_src_po": 40, "layer_head_dest_src_pos_pattern_patch_sett": 40, "get_act_patch_attn_head_q_all_po": 40, "get_act_patch_attn_head_q_by_po": 40, "get_act_patch_attn_head_v_all_po": 40, "get_act_patch_attn_head_v_by_po": 40, "get_act_patch_attn_out": 40, "layer_pos_patch_sett": 40, "get_act_patch_block_everi": 40, "get_act_patch_mlp_out": 40, "get_act_patch_resid_mid": 40, "get_act_patch_resid_pr": 40, "clean_activ": 40, "weight_convers": 41, "coder": 42, "mingpt": 42, "nanogpt": 42, "neel_solu_old": 42, "phi3": 42, "convert_bloom_weight": 44, "convert_coder_weight": 45, "convert_mistral_weight": 51, "convert_mixtral_weight": 52, "convert_phi_weight": 58, "convert_phi3_weight": 59, "convert_qwen_weight": 60, "convert_qwen2_weight": 61, "convert_t5_weight": 62, "hookedtransformertrainconfig": 63, "num_epoch": 63, "001": 63, "max_grad_norm": 63, "weight_decai": 63, "optimizer_nam": 63, "adam": 63, "warmup_step": 63, "save_everi": 63, "save_dir": 63, "wandb": 63, "wandb_project_nam": 63, "print_everi": 63, "max_step": 63, "hyperparamet": [63, 69], "epoch": 63, "decai": 63, "warmup": 63, "wandb_project": 63, "termin": 63, "activation_funct": 64, "addmm": 64, "batch_addmm": 66, "d_out": [66, 69], "d_in": [66, 69], "fuse": 66, "conv1d": 66, "9ba9369a2557e53a01378199a9839ec6e82d8bc7": 66, "src": 66, "pytorch_util": 66, "l102": 66, "l106": 66, "vanilla_addmm": 66, "mat1": 66, "mat2": 66, "typecheck": 66, "complex_attn_linear": 67, "almost": [67, 70], "simple_attn_linear": 67, "extra": [67, 69], "assist": 68, "get_device_for_block_index": 68, "target": 68, "move_to_and_update_config": 68, "vari": [69, 70], "throughout": [69, 71], "locallyoverridendefault": 69, "restor": 69, "overriden": 69, "input_slic": 69, "syntax": [69, 70, 71], "reduc": [69, 70, 71], "leav": [69, 71], "elif": 69, "1d": 69, "sliceinput": 69, "valueerror": 69, "abov": [69, 70, 71], "max_ctx": 69, "int32": 69, "slice_input": 69, "calc_fan_in_and_fan_out": 69, "fan": 69, "composition_scor": 69, "broadcast_dim": 69, "leading_dims_left_and_right": 69, "download_file_from_hf": 69, "repo_nam": 69, "file_nam": 69, "subfold": 69, "home": 69, "runner": 69, "force_is_torch": 69, "json": 69, "pth": 69, "extens": [69, 70], "layer_typ": [69, 70], "shorthand": 69, "loop": [69, 70, 71, 73], "hack": [69, 71], "stuff": [69, 71], "readabl": 69, "digit": [69, 71], "k6": 69, "scale4ln1": 69, "appear": [69, 71], "distinguish": [69, 70], "hook_k": [69, 71], "hook_pr": [69, 71], "27": [69, 70, 71], "hook_norm": [69, 71], "pre5": 69, "get_attention_mask": 69, "leftmost": 69, "rightmost": 69, "consid": 69, "get_cumsum_along_dim": 69, "dataset_nam": 69, "000": [69, 71], "enorm": [69, 71], "100gb": 69, "2tb": 69, "effort": [69, 70], "dataload": 69, "fanci": 69, "data_dir": 69, "approx": [69, 70, 71], "ton": [69, 73], "divers": [69, 70, 71], "coloss": 69, "crawl": 69, "bigger": 69, "c4_code": 69, "friendli": 69, "22m": [69, 71], "5m": 69, "20220301": 69, "en": [69, 71], "get_devic": [69, 70, 71], "get_input_with_manually_prepended_bo": 69, "autotoken": 69, "get_nested_attr": 69, "obj": 69, "attr_str": 69, "retriev": 69, "nest": 69, "hierarchi": 69, "get_offset_position_id": 69, "offset": [69, 70, 71], "get_tokenizer_with_bo": 69, "Such": [69, 70], "llamatoken": 69, "get_tokens_with_bos_remov": 69, "init_kaiming_normal_": 69, "nonlinear": 69, "std": 69, "init_kaiming_uniform_": 69, "init_xavier_normal_": 69, "init_xavier_uniform_": 69, "is_lower_triangular": 69, "is_squar": 69, "keep_single_column": 69, "col_nam": 69, "lm_accuraci": 69, "seq_len": [69, 70, 71], "altern": 69, "override_or_use_default_valu": 69, "default_flag": 69, "print_gpu_mem": 69, "step_nam": 69, "repeat_along_head_dimens": 69, "clone_tensor": 69, "sample_logit": 69, "final_logit": [69, 70], "vocab_s": 69, "argmaxi": 69, "90": 69, "renormalis": 69, "mutual": 69, "neither": [69, 70], "input_token": 69, "todo": 69, "edg": 69, "randn": [69, 71], "uniqu": 69, "return_count": 69, "set_nested_attr": 69, "prepend_space_to_answ": 69, "eleph": 69, "endoftext": [69, 70, 71], "14": [69, 70, 71, 72], "51": [69, 71], "0th": [69, 70], "59": [69, 71, 72], "ground": [69, 70], "1th": [69, 70], "41": [69, 71], "tree": 69, "2th": [69, 70], "3th": [69, 70], "45": [69, 71], "car": 69, "4th": [69, 70], "13": [69, 70, 71], "92": [69, 70], "55": [69, 70, 71], "river": 69, "5th": [69, 70], "79": 69, "25": [69, 70, 71, 72], "street": 69, "6th": [69, 70], "77": 69, "21": [69, 70, 71], "7th": [69, 70], "75": 69, "hill": 69, "8th": [69, 70], "swing": 69, "9th": [69, 70], "46": [69, 71, 72], "61": [69, 71, 72], "park": [69, 70], "to_numpi": [69, 70, 71], "tokenize_and_concaten": 69, "max_length": 69, "column_nam": 69, "num_proc": 69, "eo": [69, 71], "reshap": [69, 70], "____": 69, "drop": [69, 71], "faster": [69, 70, 71], "parallelis": [69, 71], "chop": 69, "privileg": 69, "earli": [69, 71], "cnn": [69, 71], "bos_token_id": 69, "swap": [69, 70], "runtim": [70, 71], "hardwar": [70, 71], "pane": [70, 71], "sidebar": [70, 71], "navig": [70, 71], "vscode": [70, 71], "outlin": 70, "tab": 70, "dropdown": [70, 71], "arrow": [70, 71], "page": [70, 71], "ctrl": [70, 71], "in_colab": [70, 71], "circuitsvi": [70, 71], "node": [70, 71], "curl": [70, 71], "fssl": [70, 71], "deb": [70, 71], "nodesourc": [70, 71], "setup_16": [70, 71], "sudo": [70, 71], "bash": [70, 71], "apt": [70, 71], "nodej": [70, 71], "noqa": [70, 71], "ipython": [70, 71], "get_ipython": [70, 71], "ip": [70, 71], "extension_manag": [70, 71], "autoreload": [70, 71], "functool": [70, 71], "plotli": [70, 71], "express": [70, 71], "px": [70, 71], "pio": [70, 71], "attention_head": 70, "fancy_einsum": [70, 71], "ifram": 70, "differenti": [70, 71], "simplic": 70, "imshow": [70, 71], "color_continuous_midpoint": [70, 71], "color_continuous_scal": [70, 71], "rdbu": [70, 71], "scatter": [70, 71], "xaxi": [70, 71], "yaxi": [70, 71], "caxi": [70, 71], "color": [70, 71], "principl": [70, 71, 73], "fun": [70, 71, 73], "gap": [70, 71, 73], "plai": [70, 71, 73], "flow": [70, 71, 73], "toolkit": [70, 71], "stylist": 70, "slowli": 70, "convei": 70, "tag": 70, "asid": 70, "flavour": 70, "weed": 70, "star": 70, "tagexampl": 70, "capabl": [70, 71], "interview": [70, 71], "kevin": [70, 71], "wang": 70, "twitter": 70, "thread": 70, "overview": 70, "bottl": [70, 71], "milk": [70, 71], "26": [70, 71, 72], "Their": 70, "skimp": 70, "rigour": 70, "suggest": 70, "evid": 70, "80m": [70, 71], "simplif": 70, "nbval_ignore_output": [70, 71], "stabl": 70, "example_prompt": 70, "example_answ": 70, "39": [70, 71], "lt": [70, 71], "gt": [70, 71], "09": [70, 71], "70": 70, "07": [70, 71], "38": [70, 71], "67": 70, "35": [70, 71], "54": [70, 71], "11": [70, 71, 72], "84": [70, 71], "73": 70, "hi": [70, 71], "06": 70, "her": [70, 71], "74": 70, "52": [70, 71, 72], "49": [70, 71], "jesu": 70, "97": 70, "42": [70, 71, 72], "him": 70, "subword": 70, "frequent": 70, "substr": [70, 71], "headach": 70, "annoi": [70, 71], "devot": 70, "sensibl": 70, "later": [70, 71], "wherev": 70, "flesh": 70, "prompt_format": 70, "jame": 70, "dan": 70, "sid": 70, "appl": 70, "martin": 70, "ami": 70, "drink": 70, "correct_token": 70, "insert": 70, "filler": 70, "newlin": 70, "intellig": 70, "complic": 70, "aggreg": 70, "original_logit": 70, "upon": 70, "subject": [70, 71], "logits_to_ave_logit_diff": 70, "per_prompt": 70, "answer_logit": 70, "gather": 70, "answer_logit_diff": 70, "detach": [70, 71], "decim": [70, 71], "original_average_logit_diff": 70, "3370": 70, "2020": 70, "7090": 70, "7970": 70, "7200": 70, "2810": 70, "6010": 70, "7670": 70, "552": 70, "33": [70, 71], "dive": 70, "spend": [70, 71], "engag": 70, "decent": [70, 71], "hypothes": 70, "cheat": [70, 71], "hypothesi": 70, "scienc": 70, "belief": 70, "trap": 70, "flounder": 70, "dogmat": 70, "overconfid": 70, "unwil": 70, "contradict": 70, "flinch": 70, "disconfirm": 70, "focu": 70, "primit": 70, "nearbi": 70, "came": 70, "trigram": 70, "symmetri": 70, "cancel": 70, "inhibit": 70, "spoiler": 70, "simplist": 70, "importantli": [70, 71], "perfectli": [70, 71], "final_residual_stream": 70, "eleg": 70, "particularli": 70, "aspect": 70, "nicer": 70, "inde": 70, "log_prob": 70, "log_softmax": 70, "logsumexp": 70, "decid": 70, "pronoun": 70, "refin": 70, "friendlier": 70, "answer_residual_direct": 70, "logit_diff_direct": 70, "account": 70, "w_u_fold": 70, "unigram": [70, 71], "statist": [70, 71], "opposit": 70, "hook_normalis": 70, "sub_layer_typ": 70, "final_token_residual_stream": 70, "scaled_final_token_residual_stream": 70, "average_logit_diff": 70, "residual_stack_to_logit_diff": 70, "scaled_residual_stack": 70, "fascinatingli": 70, "utterli": 70, "unabl": 70, "hover": [70, 71], "n_pre": 70, "n_mid": 70, "n_post": 70, "middl": [70, 71], "accumulated_residu": 70, "logit_lens_logit_diff": 70, "arang": 70, "hover_nam": [70, 71], "terminologi": 70, "overload": 70, "kth": 70, "per_layer_residu": 70, "per_layer_logit_diff": 70, "independ": [70, 71, 73], "l9h6": 70, "l9h9": 70, "l10h7": 70, "l11h10": 70, "harm": 70, "strongli": 70, "observ": [70, 71], "144": 70, "claim": 70, "surpris": 70, "7x": 70, "per_head_residu": 70, "per_head_logit_diff": 70, "rearrang": 70, "weren": 70, "alan": [70, 71], "coonei": [70, 71], "illustr": [70, 71], "mistak": 70, "mayb": [70, 71], "sai": [70, 71], "summari": 70, "sole": 70, "visualize_attention_pattern": 70, "local_cach": 70, "local_token": 70, "max_width": 70, "700": 70, "isinst": 70, "batch_index": 70, "combin": [70, 71], "attention_head_nam": 70, "show_cod": 70, "title_html": 70, "br": 70, "div": 70, "width": [70, 71], "top_positive_logit_attr_head": 70, "positive_html": 70, "top_negative_logit_attr_head": 70, "negative_html": 70, "conceptu": 70, "clearli": 70, "compos": [70, 71], "ideal": [70, 71], "david": [70, 71], "bau": [70, 71], "meng": [70, 71], "trace": [70, 71], "anim": 70, "lai": 70, "pro": 70, "con": 70, "Or": 70, "bake": 70, "claus": 70, "tack": 70, "gaussian": 70, "nois": 70, "beforehand": 70, "19": [70, 71], "corrupted_prompt": [70, 71], "corrupted_logit": [70, 71], "corrupted_cach": 70, "corrupted_average_logit_diff": 70, "temporarili": [70, 71], "patch_residual_compon": 70, "corrupted_residual_compon": 70, "normalize_patched_logit_diff": 70, "patched_logit_diff": [70, 71], "wors": [70, 71], "patched_residual_stream_diff": 70, "hook_fn": 70, "patched_logit": [70, 71], "abus": 70, "prompt_position_label": 70, "tok": 70, "_": [70, 71], "enumer": [70, 71], "reus": 70, "patched_attn_diff": 70, "patched_mlp_diff": 70, "patched_attn_logit": 70, "patched_attn_logit_diff": 70, "patched_mlp_logit": 70, "patched_mlp_logit_diff": 70, "late": [70, 71], "contrast": 70, "statement": 70, "mlp0": 70, "destroi": 70, "frame": 70, "unprincipl": 70, "invers": [70, 71], "plausibli": 70, "dedic": 70, "overcom": 70, "love": 70, "someon": 70, "patch_head_vector": 70, "corrupted_head_vector": 70, "patched_head_z_diff": 70, "l8h6": 70, "l8h10": 70, "l7h9": 70, "l5h5": 70, "l6h9": 70, "l3h0": 70, "semi": 70, "disentangl": 70, "familiar": 70, "28": [70, 71, 72], "patched_head_v_diff": 70, "heatmap": 70, "29": [70, 71], "lesson": 70, "head_label": 70, "range_x": 70, "range_i": 70, "31": [70, 71], "patch_head_pattern": 70, "corrupted_head_pattern": 70, "patched_head_attn_diff": 70, "32": [70, 71, 72], "reconsolid": 70, "extend": 70, "l7h3": 70, "specul": 70, "mysteri": [70, 71], "top_heads_by_output_patch": 70, "first_mid_lay": 70, "first_late_lay": 70, "early_head": 70, "mid_head": 70, "logical_and": 70, "late_head": 70, "diagram": [70, 73], "l1h2": 70, "latest": 70, "definit": 70, "priori": 70, "stroke": 70, "didn": 70, "bracket": 70, "serv": [70, 71], "particip": 70, "behav": 70, "l5h0": 70, "wrote": [70, 71, 73], "overkil": 70, "simpler": 70, "repurpos": 70, "machineri": 70, "life": [70, 71], "built": 70, "34": [70, 71], "example_text": [70, 71], "seek": 70, "machin": [70, 71], "example_repeated_text": 70, "example_repeated_token": 70, "example_repeated_logit": 70, "example_repeated_cach": 70, "induction_head_label": 70, "81": 70, "65": 70, "800": 70, "accord": 70, "wildli": 70, "characteris": 70, "superfici": 70, "boost": [70, 71], "anti": 70, "suppress": [70, 71], "pick": [70, 71], "signal": 70, "hook_": 70, "hook_attn": 70, "token_po": 70, "metadata": 70, "36": [70, 71, 72], "prev_token_scor": 70, "prev_token_hook": 70, "dim1": [70, 71], "dim2": [70, 71], "duplicate_token_scor": 70, "duplicate_token_hook": 70, "induction_scor": [70, 71], "induction_hook": 70, "manual_se": [70, 71], "original_token": 70, "randint": [70, 71], "20000": [70, 71], "repeated_token": [70, 71], "pattern_filt": 70, "act_nam": [70, 71], "endswith": [70, 71], "hook_pattern": [70, 71], "0390": 70, "0310": 70, "1890": 70, "1720": 70, "0680": 70, "1570": 70, "0210": 70, "4820": 70, "0030": 70, "1320": 70, "0050": 70, "0020": 70, "0090": 70, "0040": 70, "0010": 70, "instantli": 70, "37": [70, 71], "seen": [70, 71], "mosaic": 70, "40": [70, 71, 72], "fascin": 70, "knock": 70, "naiv": [70, 71], "convers": 70, "flaw": 70, "knockout": 70, "send": 70, "redund": 70, "job": 70, "underestim": 70, "57": [70, 71], "99": [70, 71], "hook_z": [70, 71], "top_name_mov": 70, "top_name_mover_lay": 70, "top_name_mover_head": 70, "ablate_top_head_hook": 70, "ablated_logit": 70, "ablated_cach": 70, "2f": [70, 71], "l10h10": 70, "margin": 70, "obvious": 70, "per_head_ablated_residu": 70, "per_head_ablated_logit_diff": 70, "04": [70, 71], "uniformli": [70, 71], "042": 70, "5200": 70, "4700": 70, "8200": 70, "5100": 70, "2600": 70, "1800": 70, "4300": 70, "5700": 70, "3500": 70, "2900": 70, "6800": 70, "4900": 70, "8700": 70, "4200": 70, "reader": [70, 71], "gentler": 71, "tip": 71, "development_mod": 71, "in_github": 71, "getenv": 71, "github_act": 71, "render": 71, "argh": 71, "notebook_connect": 71, "cv": 71, "hello": 71, "auto": 71, "autograd": 71, "grad_mod": 71, "0x7f0f50f9df90": 71, "speak": [71, 73], "human": [71, 73], "palm": [71, 73], "nor": [71, 73], "offend": [71, 73], "anthrop": [71, 73], "team": [71, 73], "got": [71, 73], "frustrat": [71, 73], "deepspe": [71, 73], "industri": [71, 73], "heavili": [71, 73], "credit": [71, 73], "nelson": [71, 73], "elhag": [71, 73], "chri": [71, 73], "olah": [71, 73], "model_description_text": 71, "hyper": 71, "1758": 71, "box": 71, "On": 71, "insid": 71, "kinda": 71, "gpt2_cache_no_batch_dim": 71, "gpt2_cach": 71, "gpt2_text": 71, "summar": 71, "supervis": 71, "taskspecif": 71, "gpt2_token": 71, "gpt2_logit": 71, "lock": 71, "grid": 71, "gpt2_str_token": 71, "cell": 71, "attn_hook_nam": 71, "attn_lay": 71, "gpt2_attn_cach": 71, "gpt2_attn": 71, "assert": 71, "neural": 71, "system": 71, "surgic": 71, "surround": 71, "current_activation_valu": 71, "new_activation_valu": 71, "substitut": 71, "relationship": 71, "underr": 71, "janki": 71, "shamelessli": 71, "probepoint": 71, "qualiti": 71, "head_ablation_hook": 71, "layer_to_abl": 71, "head_index_to_abl": 71, "original_loss": 71, "ablated_loss": 71, "3f": 71, "999": 71, "453": 71, "stai": 71, "clean_prompt": 71, "clean_token": 71, "logits_to_logit_diff": 71, "correct_answ": 71, "incorrect_answ": 71, "correct_index": 71, "incorrect_index": 71, "clean_logit": 71, "clean_logit_diff": 71, "corrupted_logit_diff": 71, "276": 71, "738": 71, "residual_stream_patching_hook": 71, "clean_resid_pr": 71, "num_posit": 71, "ioi_patching_result": 71, "temp_hook_fn": 71, "ish": 71, "token_label": 71, "workflow": 71, "michael": 71, "jordan": 71, "surnam": 71, "occurr": 71, "terribl": 71, "halfwai": 71, "input_tensor": 71, "random_token": 71, "repeated_logit": 71, "correct_log_prob": 71, "loss_by_posit": 71, "manipul": 71, "hook_funct": 71, "induction_score_stor": 71, "induction_score_hook": 71, "induction_strip": 71, "pattern_hook_names_filt": 71, "highli": 71, "stripe": 71, "induction_head_lay": 71, "induction_head_index": 71, "single_random_sequ": 71, "repeated_random_sequ": 71, "visualize_pattern_hook": 71, "3d": 71, "four": 71, "300m": 71, "soon": 71, "distilgpt": 71, "distilgpt2_induction_score_stor": 71, "classic": 71, "openai": 71, "85m": [71, 72], "700m": 71, "22b": 71, "300b": 71, "180b": 71, "600": 71, "265": 71, "108m": 71, "bookscorpu": 71, "free": 71, "512": [71, 72], "tractabl": 71, "motif": 71, "80": [71, 72], "shuffl": 71, "scan": 71, "40m": 71, "100m": 71, "200m": 71, "340m": [71, 72], "15b": [71, 72], "13m": [71, 72], "digress": 71, "usefulli": 71, "variengien": 71, "websit": 71, "cleantransformerdemo": 71, "new_activ": 71, "old_activ": 71, "remind": 71, "50267": 71, "named_paramet": 71, "startswith": 71, "fallback": 71, "spam": 71, "dest_posit": 71, "brown": 71, "fox": 71, "lazi": 71, "dog": 71, "num": 71, "print_name_shape_hook_funct": 71, "not_in_late_block_filt": 71, "hook_q": 71, "hook_v": 71, "hook_attn_scor": 71, "hook_attn_out": 71, "hook_resid_mid": 71, "hook_post": 71, "hook_mlp_out": 71, "hook_resid_post": 71, "preconcept": 71, "overhead": 71, "elementwis": 71, "consequ": 71, "rare": 71, "dramat": 71, "degre": 71, "punctuat": 71, "ass": 71, "randomredditor": 71, "unembed_bia": 71, "bias_valu": 71, "bias_indic": 71, "repr": 71, "03": 71, "98": 71, "68": 71, "48": [71, 72], "47": 71, "72": [71, 72], "44": [71, 72], "82": 71, "\u30b5\u30fc\u30c6\u30a3": 71, "83": 71, "x18": 71, "x14": 71, "\u9f8d": 71, "x1b": 71, "x05": 71, "x00": 71, "x06": 71, "x07": 71, "x0c": 71, "x02": 71, "oreandonlin": 71, "x11": 71, "x10": 71, "favour": 71, "6x": 71, "john_bia": 71, "mary_bia": 71, "4f": 71, "exp": 71, "8995": 71, "6034": 71, "6550x": 71, "finit": 71, "invert": 71, "de": 71, "uncommon": 71, "iz": 71, "charact": 71, "example_text_str_token": 71, "example_text_token": 71, "50256": 71, "464": 71, "717": 71, "1517": 71, "345": 71, "761": 71, "284": 71, "3785": 71, "503": 71, "318": 71, "1635": 71, "4919": 71, "1243": 71, "389": 71, "11241": 71, "1143": 71, "4600": 71, "19849": 71, "1462": 71, "62": 71, "2536": 71, "482": 71, "641": 71, "63": 71, "30778": 71, "257": 71, "4731": 71, "656": 71, "262": 71, "16326": 71, "292": 71, "1351": 71, "286": 71, "850": 71, "37336": 71, "25666": 71, "290": 71, "523": 71, "8781": 71, "7301": 71, "644": 71, "2420": 71, "3073": 71, "588": 71, "1675": 71, "10176": 71, "428": 71, "1309": 71, "338": 71, "779": 71, "340": 71, "319": 71, "7322": 71, "signifi": 71, "example_multi_text": 71, "cat": 71, "sat": 71, "mat": 71, "example_multi_text_token": 71, "3797": 71, "3332": 71, "2603": 71, "1107": 71, "1327": 71, "th": 71, "cat_text": 71, "cat_logit": 71, "cat_prob": 71, "capital_the_token_index": 71, "ascii": 71, "squeez": 71, "annoy": 71, "arithmet": 71, "impress": 71, "2342": 71, "2017": 71, "21445": 71, "1000000": 71, "999999": 71, "214": 71, "000000": 71, "9999": 71, "tim": 71, "ne": 71, "el": 71, "messier": 71, "takeawai": 71, "unexpect": 71, "notic": 71, "trip": 71, "confusingli": 71, "forth": 71, "ioi_logits_with_bo": 71, "clair": 71, "mary_logit_with_bo": 71, "claire_logit_with_bo": 71, "ioi_logits_without_bo": 71, "mary_logit_without_bo": 71, "claire_logit_without_bo": 71, "754": 71, "782": 71, "air": 71, "understood": 71, "requisit": 71, "attention_scor": 71, "ab_factor": 71, "9105": 71, "linalg": 71, "eig": 71, "2877e": 71, "00": 71, "8626e": 71, "3121e": 71, "9038e": 71, "08": 71, "1527e": 71, "2877": 71, "3121": 71, "3126e": 71, "3963e": 71, "2029e": 71, "7690e": 71, "2164e": 71, "3126": 71, "3963": 71, "43": 71, "300": 71, "abc_factor": 71, "unfactor": 71, "160": 71, "0830": 71, "ab_unfactor": 71, "isclos": 71, "subspac": 71, "coincid": 71, "negat": 71, "proxi": 71, "lambda_i": 71, "ov_circuit_all_head": 71, "ov_circuit_all_heads_eigenvalu": 71, "complex64": 71, "ov_copying_scor": 71, "zmax": 71, "zmin": 71, "l11h11": 71, "imag": 71, "imaginari": 71, "full_ov_circuit": 71, "full_ov_circuit_eigenvalu": 71, "full_ov_copying_scor": 71, "interestingli": 71, "correl": 71, "outlier": 71, "53": 71, "ansh": 71, "radhakrishnan": 71, "establish": 71, "presid": 71, "barack": 71, "obama": 71, "caught": 71, "embarrass": 71, "scandal": 71, "nthe": 71, "financi": 71, "wife": 71, "chelsea": 71, "she": 71, "woman": 71, "lightweight": 71, "squarethenadd": 71, "hook_squar": 71, "twolayermodel": 71, "layer1": 71, "layer2": 71, "hook_in": 71, "hook_mid": 71, "hook_out": 71, "x_in": 71, "x_mid": 71, "x_out": 71, "model_out": 71, "cache_object": 71, "56": [71, 72], "780": 71, "784": 71, "set_to_zero_hook": 71, "num_checkpoint": 71, "piecewis": 71, "schedul": 71, "crash": 71, "11b": [71, 72], "centr": 71, "hoc": 71, "count": 71, "58": 71, "checkpoint_label": 71, "log_i": 71, "marker": 71, "brief": 71, "suddenli": 71, "500": 71, "visibl": 71, "curv": 71, "briefli": 71, "deliber": 71, "justic": 71, "chosen": 71, "60": [71, 72], "500m": 71, "arbitrarili": 71, "fast": 71, "checkpoint_indic": 71, "checkpointed_model": 71, "tokens_trained_on": 71, "model_for_this_checkpoint": 71, "tokens_seen_for_this_checkpoint": 71, "induction_loss_for_this_checkpoint": 71, "contextualis": 71, "95": 71, "log_x": 71, "302m": 72, "4096": 72, "708m": 72, "1280": 72, "5120": 72, "1600": 72, "6400": 72, "42m": 72, "2048": 72, "50272": 72, "8192": 72, "2560": 72, "10240": 72, "128": 72, "16384": 72, "20480": 72, "7168": 72, "28672": 72, "9216": 72, "36864": 72, "50400": 72, "6144": 72, "50432": 72, "96": 72, "24576": 72, "2m": 72, "50304": 72, "7m": 72, "805m": 72, "50688": 72, "50278": 72, "736": 72, "2944": 72, "101m": 72, "197m": 72, "1536": 72, "48262": 72, "4m": 72, "0m": 72, "50277": 72, "524k": 72, "50259": 72, "32000": 72, "11008": 72, "13824": 72, "6656": 72, "17920": 72, "22016": 72, "78b": 72, "32016": 72, "128256": 72, "14336": 72, "25m": 72, "28996": 72, "393k": 72, "6m": 72, "131072": 72, "47b": 72, "32768": 72, "250880": 72, "679m": 72, "0b": 72, "49280": 72, "151936": 72, "5504": 72, "152064": 72, "13696": 72, "308m": 72, "2816": 72, "6912": 72, "391m": 72, "896": 72, "4864": 72, "8960": 72, "3584": 72, "18944": 72, "27648": 72, "80b": 72, "29568": 72, "51200": 72, "32064": 72, "256000": 72, "2304": 72, "gelu_pytorch_tanh": 72, "4608": 72, "64000": 72, "39b": 72, "32128": 72, "100000": 72, "formerli": 73, "transfer": 73, "courtesi": 73, "austin": 73, "kozlowski": 73}, "objects": {"transformer_lens": [[10, 0, 0, "-", "ActivationCache"], [11, 0, 0, "-", "FactoredMatrix"], [12, 0, 0, "-", "HookedEncoder"], [13, 0, 0, "-", "HookedEncoderDecoder"], [14, 0, 0, "-", "HookedTransformer"], [15, 0, 0, "-", "HookedTransformerConfig"], [16, 0, 0, "-", "SVDInterpreter"], [35, 0, 0, "-", "evals"], [36, 0, 0, "-", "head_detector"], [37, 0, 0, "-", "hook_points"], [38, 0, 0, "-", "loading_from_pretrained"], [39, 0, 0, "-", "past_key_value_caching"], [40, 0, 0, "-", "patching"], [63, 0, 0, "-", "train"], [69, 0, 0, "-", "utils"]], "transformer_lens.ActivationCache": [[10, 1, 1, "", "ActivationCache"]], "transformer_lens.ActivationCache.ActivationCache": [[10, 2, 1, "", "accumulated_resid"], [10, 2, 1, "", "apply_ln_to_stack"], [10, 2, 1, "", "apply_slice_to_batch_dim"], [10, 2, 1, "", "compute_head_results"], [10, 2, 1, "", "decompose_resid"], [10, 2, 1, "", "get_full_resid_decomposition"], [10, 2, 1, "", "get_neuron_results"], [10, 2, 1, "", "items"], [10, 2, 1, "", "keys"], [10, 2, 1, "", "logit_attrs"], [10, 2, 1, "", "remove_batch_dim"], [10, 2, 1, "", "stack_activation"], [10, 2, 1, "", "stack_head_results"], [10, 2, 1, "", "stack_neuron_results"], [10, 2, 1, "", "to"], [10, 2, 1, "", "toggle_autodiff"], [10, 2, 1, "", "values"]], "transformer_lens.FactoredMatrix": [[11, 1, 1, "", "FactoredMatrix"]], "transformer_lens.FactoredMatrix.FactoredMatrix": [[11, 3, 1, "", "AB"], [11, 3, 1, "", "BA"], [11, 3, 1, "", "S"], [11, 3, 1, "", "T"], [11, 3, 1, "", "U"], [11, 3, 1, "", "Vh"], [11, 2, 1, "", "collapse_l"], [11, 2, 1, "", "collapse_r"], [11, 3, 1, "", "eigenvalues"], [11, 2, 1, "", "get_corner"], [11, 2, 1, "", "make_even"], [11, 3, 1, "", "ndim"], [11, 2, 1, "", "norm"], [11, 3, 1, "", "pair"], [11, 2, 1, "", "svd"], [11, 2, 1, "", "unsqueeze"]], "transformer_lens.HookedEncoder": [[12, 1, 1, "", "HookedEncoder"]], "transformer_lens.HookedEncoder.HookedEncoder": [[12, 3, 1, "", "OV"], [12, 3, 1, "", "QK"], [12, 3, 1, "", "W_E"], [12, 3, 1, "", "W_E_pos"], [12, 3, 1, "", "W_K"], [12, 3, 1, "", "W_O"], [12, 3, 1, "", "W_Q"], [12, 3, 1, "", "W_U"], [12, 3, 1, "", "W_V"], [12, 3, 1, "", "W_in"], [12, 3, 1, "", "W_out"], [12, 3, 1, "", "W_pos"], [12, 2, 1, "", "all_head_labels"], [12, 3, 1, "", "b_K"], [12, 3, 1, "", "b_O"], [12, 3, 1, "", "b_Q"], [12, 3, 1, "", "b_U"], [12, 3, 1, "", "b_V"], [12, 3, 1, "", "b_in"], [12, 3, 1, "", "b_out"], [12, 2, 1, "", "cpu"], [12, 2, 1, "", "cuda"], [12, 2, 1, "", "forward"], [12, 2, 1, "", "from_pretrained"], [12, 2, 1, "", "mps"], [12, 2, 1, "", "run_with_cache"], [12, 2, 1, "", "to"]], "transformer_lens.HookedEncoderDecoder": [[13, 1, 1, "", "HookedEncoderDecoder"]], "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder": [[13, 3, 1, "", "OV"], [13, 3, 1, "", "QK"], [13, 3, 1, "", "W_E"], [13, 3, 1, "", "W_K"], [13, 3, 1, "", "W_O"], [13, 3, 1, "", "W_Q"], [13, 3, 1, "", "W_U"], [13, 3, 1, "", "W_V"], [13, 3, 1, "", "W_in"], [13, 3, 1, "", "W_out"], [13, 3, 1, "", "W_pos"], [13, 2, 1, "", "all_head_labels"], [13, 3, 1, "", "b_K"], [13, 3, 1, "", "b_O"], [13, 3, 1, "", "b_Q"], [13, 3, 1, "", "b_U"], [13, 3, 1, "", "b_V"], [13, 3, 1, "", "b_in"], [13, 3, 1, "", "b_out"], [13, 2, 1, "", "cpu"], [13, 2, 1, "", "cuda"], [13, 2, 1, "", "forward"], [13, 2, 1, "", "from_pretrained"], [13, 2, 1, "", "mps"], [13, 2, 1, "", "run_with_cache"], [13, 2, 1, "", "to"]], "transformer_lens.HookedTransformer": [[14, 1, 1, "", "HookedTransformer"], [14, 1, 1, "", "Output"]], "transformer_lens.HookedTransformer.HookedTransformer": [[14, 3, 1, "", "OV"], [14, 3, 1, "", "QK"], [14, 3, 1, "", "W_E"], [14, 3, 1, "", "W_E_pos"], [14, 3, 1, "", "W_K"], [14, 3, 1, "", "W_O"], [14, 3, 1, "", "W_Q"], [14, 3, 1, "", "W_U"], [14, 3, 1, "", "W_V"], [14, 3, 1, "", "W_gate"], [14, 3, 1, "", "W_in"], [14, 3, 1, "", "W_out"], [14, 3, 1, "", "W_pos"], [14, 2, 1, "", "__init__"], [14, 2, 1, "", "accumulated_bias"], [14, 2, 1, "", "all_composition_scores"], [14, 2, 1, "", "all_head_labels"], [14, 3, 1, "", "b_K"], [14, 3, 1, "", "b_O"], [14, 3, 1, "", "b_Q"], [14, 3, 1, "", "b_U"], [14, 3, 1, "", "b_V"], [14, 3, 1, "", "b_in"], [14, 3, 1, "", "b_out"], [14, 2, 1, "", "center_unembed"], [14, 2, 1, "", "center_writing_weights"], [14, 2, 1, "", "check_hooks_to_add"], [14, 2, 1, "", "cpu"], [14, 2, 1, "", "cuda"], [14, 2, 1, "", "fold_layer_norm"], [14, 2, 1, "", "fold_value_biases"], [14, 2, 1, "", "forward"], [14, 2, 1, "", "from_pretrained"], [14, 2, 1, "", "from_pretrained_no_processing"], [14, 2, 1, "", "generate"], [14, 2, 1, "", "get_token_position"], [14, 2, 1, "", "init_weights"], [14, 2, 1, "", "input_to_embed"], [14, 4, 1, "", "ln_final"], [14, 2, 1, "", "load_and_process_state_dict"], [14, 2, 1, "", "load_sample_training_dataset"], [14, 2, 1, "", "loss_fn"], [14, 2, 1, "", "move_model_modules_to_device"], [14, 2, 1, "", "mps"], [14, 2, 1, "", "process_weights_"], [14, 2, 1, "", "refactor_factored_attn_matrices"], [14, 2, 1, "", "run_with_cache"], [14, 2, 1, "", "sample_datapoint"], [14, 2, 1, "", "set_tokenizer"], [14, 2, 1, "", "set_ungroup_grouped_query_attention"], [14, 2, 1, "", "set_use_attn_in"], [14, 2, 1, "", "set_use_attn_result"], [14, 2, 1, "", "set_use_hook_mlp_in"], [14, 2, 1, "", "set_use_split_qkv_input"], [14, 2, 1, "", "to"], [14, 2, 1, "", "to_single_str_token"], [14, 2, 1, "", "to_single_token"], [14, 2, 1, "", "to_str_tokens"], [14, 2, 1, "", "to_string"], [14, 2, 1, "", "to_tokens"], [14, 2, 1, "", "tokens_to_residual_directions"]], "transformer_lens.HookedTransformer.Output": [[14, 4, 1, "", "logits"], [14, 4, 1, "", "loss"]], "transformer_lens.HookedTransformerConfig": [[15, 1, 1, "", "HookedTransformerConfig"]], "transformer_lens.HookedTransformerConfig.HookedTransformerConfig": [[15, 4, 1, "", "NTK_by_parts_factor"], [15, 4, 1, "", "NTK_by_parts_high_freq_factor"], [15, 4, 1, "", "NTK_by_parts_low_freq_factor"], [15, 4, 1, "", "act_fn"], [15, 4, 1, "", "attention_dir"], [15, 4, 1, "", "attn_only"], [15, 4, 1, "", "attn_scale"], [15, 4, 1, "", "attn_scores_soft_cap"], [15, 4, 1, "", "attn_types"], [15, 4, 1, "", "checkpoint_index"], [15, 4, 1, "", "checkpoint_label_type"], [15, 4, 1, "", "checkpoint_value"], [15, 4, 1, "", "d_head"], [15, 4, 1, "", "d_mlp"], [15, 4, 1, "", "d_model"], [15, 4, 1, "", "d_vocab"], [15, 4, 1, "", "d_vocab_out"], [15, 4, 1, "", "decoder_start_token_id"], [15, 4, 1, "", "default_prepend_bos"], [15, 4, 1, "", "device"], [15, 4, 1, "", "dtype"], [15, 4, 1, "", "eps"], [15, 4, 1, "", "experts_per_token"], [15, 4, 1, "", "final_rms"], [15, 4, 1, "", "from_checkpoint"], [15, 2, 1, "", "from_dict"], [15, 4, 1, "", "gated_mlp"], [15, 4, 1, "", "init_mode"], [15, 4, 1, "", "init_weights"], [15, 4, 1, "", "initializer_range"], [15, 2, 1, "", "is_layer_norm_activation"], [15, 4, 1, "", "load_in_4bit"], [15, 4, 1, "", "model_name"], [15, 4, 1, "", "n_ctx"], [15, 4, 1, "", "n_devices"], [15, 4, 1, "", "n_heads"], [15, 4, 1, "", "n_key_value_heads"], [15, 4, 1, "", "n_layers"], [15, 4, 1, "", "n_params"], [15, 4, 1, "", "normalization_type"], [15, 4, 1, "", "num_experts"], [15, 4, 1, "", "original_architecture"], [15, 4, 1, "", "output_logits_soft_cap"], [15, 4, 1, "", "parallel_attn_mlp"], [15, 4, 1, "", "positional_embedding_type"], [15, 4, 1, "", "post_embedding_ln"], [15, 4, 1, "", "relative_attention_max_distance"], [15, 4, 1, "", "relative_attention_num_buckets"], [15, 4, 1, "", "rotary_adjacent_pairs"], [15, 4, 1, "", "rotary_base"], [15, 4, 1, "", "rotary_dim"], [15, 4, 1, "", "scale_attn_by_inverse_layer_idx"], [15, 4, 1, "", "seed"], [15, 2, 1, "", "set_seed_everywhere"], [15, 4, 1, "", "tie_word_embeddings"], [15, 2, 1, "", "to_dict"], [15, 4, 1, "", "tokenizer_name"], [15, 4, 1, "", "tokenizer_prepends_bos"], [15, 4, 1, "", "trust_remote_code"], [15, 4, 1, "", "ungroup_grouped_query_attention"], [15, 2, 1, "", "unwrap"], [15, 4, 1, "", "use_NTK_by_parts_rope"], [15, 4, 1, "", "use_attn_in"], [15, 4, 1, "", "use_attn_result"], [15, 4, 1, "", "use_attn_scale"], [15, 4, 1, "", "use_hook_mlp_in"], [15, 4, 1, "", "use_hook_tokens"], [15, 4, 1, "", "use_local_attn"], [15, 4, 1, "", "use_normalization_before_and_after"], [15, 4, 1, "", "use_split_qkv_input"], [15, 4, 1, "", "window_size"]], "transformer_lens.SVDInterpreter": [[16, 1, 1, "", "SVDInterpreter"]], "transformer_lens.SVDInterpreter.SVDInterpreter": [[16, 2, 1, "", "get_singular_vectors"]], "transformer_lens.components": [[18, 0, 0, "-", "abstract_attention"], [19, 0, 0, "-", "attention"], [20, 0, 0, "-", "bert_block"], [21, 0, 0, "-", "bert_embed"], [22, 0, 0, "-", "bert_mlm_head"], [23, 0, 0, "-", "embed"], [24, 0, 0, "-", "grouped_query_attention"], [25, 0, 0, "-", "layer_norm"], [26, 0, 0, "-", "layer_norm_pre"], [27, 0, 0, "-", "pos_embed"], [28, 0, 0, "-", "rms_norm"], [29, 0, 0, "-", "rms_norm_pre"], [30, 0, 0, "-", "t5_attention"], [31, 0, 0, "-", "t5_block"], [32, 0, 0, "-", "token_typed_embed"], [33, 0, 0, "-", "transformer_block"], [34, 0, 0, "-", "unembed"]], "transformer_lens.components.abstract_attention": [[18, 1, 1, "", "AbstractAttention"]], "transformer_lens.components.abstract_attention.AbstractAttention": [[18, 3, 1, "", "OV"], [18, 3, 1, "", "QK"], [18, 2, 1, "", "__init__"], [18, 4, 1, "", "alibi"], [18, 2, 1, "", "apply_causal_mask"], [18, 2, 1, "", "apply_rotary"], [18, 2, 1, "", "calculate_attention_scores"], [18, 2, 1, "", "calculate_qkv_matrices"], [18, 2, 1, "", "calculate_sin_cos_rotary"], [18, 2, 1, "", "calculate_z_scores"], [18, 2, 1, "", "create_alibi_bias"], [18, 2, 1, "", "create_alibi_multipliers"], [18, 2, 1, "", "create_alibi_slope"], [18, 2, 1, "", "forward"], [18, 2, 1, "", "rotate_every_two"]], "transformer_lens.components.attention": [[19, 1, 1, "", "Attention"]], "transformer_lens.components.attention.Attention": [[19, 2, 1, "", "__init__"]], "transformer_lens.components.bert_block": [[20, 1, 1, "", "BertBlock"]], "transformer_lens.components.bert_block.BertBlock": [[20, 2, 1, "", "forward"]], "transformer_lens.components.bert_embed": [[21, 1, 1, "", "BertEmbed"]], "transformer_lens.components.bert_embed.BertEmbed": [[21, 2, 1, "", "forward"]], "transformer_lens.components.bert_mlm_head": [[22, 1, 1, "", "BertMLMHead"]], "transformer_lens.components.bert_mlm_head.BertMLMHead": [[22, 2, 1, "", "forward"]], "transformer_lens.components.embed": [[23, 1, 1, "", "Embed"]], "transformer_lens.components.embed.Embed": [[23, 2, 1, "", "forward"]], "transformer_lens.components.grouped_query_attention": [[24, 1, 1, "", "GroupedQueryAttention"]], "transformer_lens.components.grouped_query_attention.GroupedQueryAttention": [[24, 3, 1, "", "W_K"], [24, 3, 1, "", "W_V"], [24, 2, 1, "", "__init__"], [24, 3, 1, "", "b_K"], [24, 3, 1, "", "b_V"], [24, 2, 1, "", "calculate_attention_scores"], [24, 2, 1, "", "calculate_qkv_matrices"], [24, 2, 1, "", "calculate_z_scores"]], "transformer_lens.components.layer_norm": [[25, 1, 1, "", "LayerNorm"]], "transformer_lens.components.layer_norm.LayerNorm": [[25, 2, 1, "", "__init__"], [25, 2, 1, "", "forward"]], "transformer_lens.components.layer_norm_pre": [[26, 1, 1, "", "LayerNormPre"]], "transformer_lens.components.layer_norm_pre.LayerNormPre": [[26, 2, 1, "", "__init__"], [26, 2, 1, "", "forward"]], "transformer_lens.components.pos_embed": [[27, 1, 1, "", "PosEmbed"]], "transformer_lens.components.pos_embed.PosEmbed": [[27, 2, 1, "", "forward"]], "transformer_lens.components.rms_norm": [[28, 1, 1, "", "RMSNorm"]], "transformer_lens.components.rms_norm.RMSNorm": [[28, 2, 1, "", "__init__"], [28, 2, 1, "", "forward"]], "transformer_lens.components.rms_norm_pre": [[29, 1, 1, "", "RMSNormPre"]], "transformer_lens.components.rms_norm_pre.RMSNormPre": [[29, 2, 1, "", "__init__"], [29, 2, 1, "", "forward"]], "transformer_lens.components.t5_attention": [[30, 1, 1, "", "T5Attention"]], "transformer_lens.components.t5_attention.T5Attention": [[30, 2, 1, "", "compute_relative_attention_bias"]], "transformer_lens.components.t5_block": [[31, 1, 1, "", "T5Block"]], "transformer_lens.components.t5_block.T5Block": [[31, 2, 1, "", "forward"]], "transformer_lens.components.token_typed_embed": [[32, 1, 1, "", "TokenTypeEmbed"]], "transformer_lens.components.token_typed_embed.TokenTypeEmbed": [[32, 2, 1, "", "forward"]], "transformer_lens.components.transformer_block": [[33, 1, 1, "", "TransformerBlock"]], "transformer_lens.components.transformer_block.TransformerBlock": [[33, 2, 1, "", "apply_mlp"], [33, 2, 1, "", "forward"], [33, 4, 1, "", "ln1"], [33, 4, 1, "", "ln2"], [33, 4, 1, "", "mlp"]], "transformer_lens.components.unembed": [[34, 1, 1, "", "Unembed"]], "transformer_lens.components.unembed.Unembed": [[34, 2, 1, "", "forward"]], "transformer_lens.evals": [[35, 1, 1, "", "IOIDataset"], [35, 5, 1, "", "evaluate"], [35, 5, 1, "", "evaluate_on_dataset"], [35, 5, 1, "", "induction_loss"], [35, 5, 1, "", "ioi_eval"], [35, 5, 1, "", "make_code_data_loader"], [35, 5, 1, "", "make_owt_data_loader"], [35, 5, 1, "", "make_pile_data_loader"], [35, 5, 1, "", "make_wiki_data_loader"], [35, 5, 1, "", "sanity_check"]], "transformer_lens.evals.IOIDataset": [[35, 2, 1, "", "get_default_names"], [35, 2, 1, "", "get_default_nouns"], [35, 2, 1, "", "get_default_templates"], [35, 2, 1, "", "get_sample"]], "transformer_lens.head_detector": [[36, 5, 1, "", "compute_head_attention_similarity_score"], [36, 5, 1, "", "detect_head"], [36, 5, 1, "", "get_duplicate_token_head_detection_pattern"], [36, 5, 1, "", "get_induction_head_detection_pattern"], [36, 5, 1, "", "get_previous_token_head_detection_pattern"], [36, 5, 1, "", "get_supported_heads"]], "transformer_lens.hook_points": [[37, 4, 1, "", "HookFunction"], [37, 1, 1, "", "HookPoint"], [37, 1, 1, "", "HookedRootModule"], [37, 1, 1, "", "LensHandle"]], "transformer_lens.hook_points.HookPoint": [[37, 2, 1, "", "add_hook"], [37, 2, 1, "", "add_perma_hook"], [37, 2, 1, "", "clear_context"], [37, 2, 1, "", "forward"], [37, 2, 1, "", "layer"], [37, 2, 1, "", "remove_hooks"]], "transformer_lens.hook_points.HookedRootModule": [[37, 2, 1, "", "add_caching_hooks"], [37, 2, 1, "", "add_hook"], [37, 2, 1, "", "add_perma_hook"], [37, 2, 1, "", "cache_all"], [37, 2, 1, "", "cache_some"], [37, 2, 1, "", "check_and_add_hook"], [37, 2, 1, "", "check_hooks_to_add"], [37, 2, 1, "", "clear_contexts"], [37, 2, 1, "", "get_caching_hooks"], [37, 4, 1, "", "hook_dict"], [37, 2, 1, "", "hook_points"], [37, 2, 1, "", "hooks"], [37, 4, 1, "", "mod_dict"], [37, 4, 1, "", "name"], [37, 2, 1, "", "remove_all_hook_fns"], [37, 2, 1, "", "reset_hooks"], [37, 2, 1, "", "run_with_cache"], [37, 2, 1, "", "run_with_hooks"], [37, 2, 1, "", "setup"]], "transformer_lens.hook_points.LensHandle": [[37, 4, 1, "", "context_level"], [37, 4, 1, "", "hook"], [37, 4, 1, "", "is_permanent"]], "transformer_lens.loading_from_pretrained": [[38, 1, 1, "", "Config"], [38, 6, 1, "", "MODEL_ALIASES"], [38, 6, 1, "", "NON_HF_HOSTED_MODEL_NAMES"], [38, 6, 1, "", "OFFICIAL_MODEL_NAMES"], [38, 5, 1, "", "get_checkpoint_labels"], [38, 5, 1, "", "get_num_params_of_pretrained"], [38, 5, 1, "", "get_pretrained_model_config"]], "transformer_lens.loading_from_pretrained.Config": [[38, 4, 1, "", "d_head"], [38, 4, 1, "", "d_mlp"], [38, 4, 1, "", "d_model"], [38, 4, 1, "", "d_vocab"], [38, 4, 1, "", "debug"], [38, 4, 1, "", "init_range"], [38, 4, 1, "", "layer_norm_eps"], [38, 4, 1, "", "n_ctx"], [38, 4, 1, "", "n_heads"], [38, 4, 1, "", "n_layers"]], "transformer_lens.past_key_value_caching": [[39, 1, 1, "", "HookedTransformerKeyValueCache"], [39, 1, 1, "", "HookedTransformerKeyValueCacheEntry"]], "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache": [[39, 2, 1, "", "append_attention_mask"], [39, 4, 1, "", "entries"], [39, 2, 1, "", "freeze"], [39, 4, 1, "", "frozen"], [39, 2, 1, "", "init_cache"], [39, 4, 1, "", "previous_attention_mask"], [39, 2, 1, "", "unfreeze"]], "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry": [[39, 2, 1, "", "append"], [39, 4, 1, "", "frozen"], [39, 2, 1, "", "init_cache_entry"], [39, 4, 1, "", "past_keys"], [39, 4, 1, "", "past_values"]], "transformer_lens.patching": [[40, 5, 1, "", "generic_activation_patch"], [40, 5, 1, "", "get_act_patch_attn_head_all_pos_every"], [40, 5, 1, "", "get_act_patch_attn_head_by_pos_every"], [40, 5, 1, "", "get_act_patch_attn_head_k_all_pos"], [40, 5, 1, "", "get_act_patch_attn_head_k_by_pos"], [40, 5, 1, "", "get_act_patch_attn_head_out_all_pos"], [40, 5, 1, "", "get_act_patch_attn_head_out_by_pos"], [40, 5, 1, "", "get_act_patch_attn_head_pattern_all_pos"], [40, 5, 1, "", "get_act_patch_attn_head_pattern_by_pos"], [40, 5, 1, "", "get_act_patch_attn_head_pattern_dest_src_pos"], [40, 5, 1, "", "get_act_patch_attn_head_q_all_pos"], [40, 5, 1, "", "get_act_patch_attn_head_q_by_pos"], [40, 5, 1, "", "get_act_patch_attn_head_v_all_pos"], [40, 5, 1, "", "get_act_patch_attn_head_v_by_pos"], [40, 5, 1, "", "get_act_patch_attn_out"], [40, 5, 1, "", "get_act_patch_block_every"], [40, 5, 1, "", "get_act_patch_mlp_out"], [40, 5, 1, "", "get_act_patch_resid_mid"], [40, 5, 1, "", "get_act_patch_resid_pre"], [40, 5, 1, "", "layer_head_dest_src_pos_pattern_patch_setter"], [40, 5, 1, "", "layer_head_pattern_patch_setter"], [40, 5, 1, "", "layer_head_pos_pattern_patch_setter"], [40, 5, 1, "", "layer_head_vector_patch_setter"], [40, 5, 1, "", "layer_pos_head_vector_patch_setter"], [40, 5, 1, "", "layer_pos_patch_setter"]], "transformer_lens.pretrained.weight_conversions": [[43, 0, 0, "-", "bert"], [44, 0, 0, "-", "bloom"], [45, 0, 0, "-", "coder"], [46, 0, 0, "-", "gemma"], [47, 0, 0, "-", "gpt2"], [48, 0, 0, "-", "gptj"], [49, 0, 0, "-", "llama"], [50, 0, 0, "-", "mingpt"], [51, 0, 0, "-", "mistral"], [52, 0, 0, "-", "mixtral"], [53, 0, 0, "-", "nanogpt"], [54, 0, 0, "-", "neel_solu_old"], [55, 0, 0, "-", "neo"], [56, 0, 0, "-", "neox"], [57, 0, 0, "-", "opt"], [58, 0, 0, "-", "phi"], [59, 0, 0, "-", "phi3"], [60, 0, 0, "-", "qwen"], [61, 0, 0, "-", "qwen2"], [62, 0, 0, "-", "t5"]], "transformer_lens.pretrained.weight_conversions.bloom": [[44, 5, 1, "", "convert_bloom_weights"]], "transformer_lens.pretrained.weight_conversions.coder": [[45, 5, 1, "", "convert_coder_weights"]], "transformer_lens.pretrained.weight_conversions.mistral": [[51, 5, 1, "", "convert_mistral_weights"]], "transformer_lens.pretrained.weight_conversions.mixtral": [[52, 5, 1, "", "convert_mixtral_weights"]], "transformer_lens.pretrained.weight_conversions.phi": [[58, 5, 1, "", "convert_phi_weights"]], "transformer_lens.pretrained.weight_conversions.phi3": [[59, 5, 1, "", "convert_phi3_weights"]], "transformer_lens.pretrained.weight_conversions.qwen": [[60, 5, 1, "", "convert_qwen_weights"]], "transformer_lens.pretrained.weight_conversions.qwen2": [[61, 5, 1, "", "convert_qwen2_weights"]], "transformer_lens.pretrained.weight_conversions.t5": [[62, 5, 1, "", "convert_t5_weights"]], "transformer_lens.train": [[63, 1, 1, "", "HookedTransformerTrainConfig"], [63, 5, 1, "", "train"]], "transformer_lens.train.HookedTransformerTrainConfig": [[63, 4, 1, "", "batch_size"], [63, 4, 1, "", "device"], [63, 4, 1, "", "lr"], [63, 4, 1, "", "max_grad_norm"], [63, 4, 1, "", "max_steps"], [63, 4, 1, "", "momentum"], [63, 4, 1, "", "num_epochs"], [63, 4, 1, "", "optimizer_name"], [63, 4, 1, "", "print_every"], [63, 4, 1, "", "save_dir"], [63, 4, 1, "", "save_every"], [63, 4, 1, "", "seed"], [63, 4, 1, "", "wandb"], [63, 4, 1, "", "wandb_project_name"], [63, 4, 1, "", "warmup_steps"], [63, 4, 1, "", "weight_decay"]], "transformer_lens.utilities": [[65, 0, 0, "-", "activation_functions"], [66, 0, 0, "-", "addmm"], [67, 0, 0, "-", "attention"], [68, 0, 0, "-", "devices"]], "transformer_lens.utilities.addmm": [[66, 5, 1, "", "batch_addmm"], [66, 5, 1, "", "vanilla_addmm"]], "transformer_lens.utilities.attention": [[67, 5, 1, "", "complex_attn_linear"], [67, 5, 1, "", "simple_attn_linear"]], "transformer_lens.utilities.devices": [[68, 5, 1, "", "get_device_for_block_index"], [68, 5, 1, "", "move_to_and_update_config"]], "transformer_lens.utils": [[69, 1, 1, "", "LocallyOverridenDefaults"], [69, 1, 1, "", "Slice"], [69, 6, 1, "", "SliceInput"], [69, 5, 1, "", "calc_fan_in_and_fan_out"], [69, 5, 1, "", "composition_scores"], [69, 5, 1, "", "download_file_from_hf"], [69, 5, 1, "", "gelu_fast"], [69, 5, 1, "", "gelu_new"], [69, 5, 1, "", "get_act_name"], [69, 5, 1, "", "get_attention_mask"], [69, 5, 1, "", "get_corner"], [69, 5, 1, "", "get_cumsum_along_dim"], [69, 5, 1, "", "get_dataset"], [69, 5, 1, "", "get_device"], [69, 5, 1, "", "get_input_with_manually_prepended_bos"], [69, 5, 1, "", "get_nested_attr"], [69, 5, 1, "", "get_offset_position_ids"], [69, 5, 1, "", "get_tokenizer_with_bos"], [69, 5, 1, "", "get_tokens_with_bos_removed"], [69, 5, 1, "", "init_kaiming_normal_"], [69, 5, 1, "", "init_kaiming_uniform_"], [69, 5, 1, "", "init_xavier_normal_"], [69, 5, 1, "", "init_xavier_uniform_"], [69, 5, 1, "", "is_lower_triangular"], [69, 5, 1, "", "is_square"], [69, 5, 1, "", "keep_single_column"], [69, 5, 1, "", "lm_accuracy"], [69, 5, 1, "", "lm_cross_entropy_loss"], [69, 5, 1, "", "override_or_use_default_value"], [69, 5, 1, "", "print_gpu_mem"], [69, 5, 1, "", "remove_batch_dim"], [69, 5, 1, "", "repeat_along_head_dimension"], [69, 5, 1, "", "sample_logits"], [69, 5, 1, "", "set_nested_attr"], [69, 5, 1, "", "solu"], [69, 5, 1, "", "test_prompt"], [69, 5, 1, "", "to_numpy"], [69, 5, 1, "", "tokenize_and_concatenate"], [69, 5, 1, "", "transpose"]], "transformer_lens.utils.LocallyOverridenDefaults": [[69, 2, 1, "", "__init__"]], "transformer_lens.utils.Slice": [[69, 2, 1, "", "__init__"], [69, 2, 1, "", "apply"], [69, 2, 1, "", "indices"], [69, 4, 1, "", "slice"], [69, 2, 1, "", "unwrap"]]}, "objtypes": {"0": "py:module", "1": "py:class", "2": "py:method", "3": "py:property", "4": "py:attribute", "5": "py:function", "6": "py:data"}, "objnames": {"0": ["py", "module", "Python module"], "1": ["py", "class", "Python class"], "2": ["py", "method", "Python method"], "3": ["py", "property", "Python property"], "4": ["py", "attribute", "Python attribute"], "5": ["py", "function", "Python function"], "6": ["py", "data", "Python data"]}, "titleterms": {"citat": 0, "contribut": 1, "setup": [1, 70, 71], "devcontain": 1, "manual": 1, "test": [1, 5], "run": [1, 71], "format": 1, "document": 1, "docstr": 1, "style": 1, "guid": 1, "section": 1, "order": 1, "support": 1, "sphinx": 1, "properti": [1, 72], "refer": 1, "other": [1, 71], "function": [1, 70], "class": [1, 71], "math": 1, "markup": 1, "galleri": 2, "get": [3, 4], "start": [3, 4, 7], "advic": 3, "read": [3, 70], "code": 3, "instal": 3, "huggingfac": 3, "gate": 3, "access": [3, 71], "mechanist": [4, 73], "interpret": [4, 71, 73], "transformerlen": [5, 73], "2": 5, "0": 5, "first": 5, "an": [5, 71], "introduct": [5, 70, 71], "adopt": 5, "semant": 5, "version": 5, "deprec": 5, "roadmap": 5, "immedi": 5, "within": 5, "next": 5, "month": 5, "mid": 5, "term": 5, "3": 5, "perform": 5, "streamlin": 5, "ad": 5, "new": 5, "model": [5, 71, 72, 73], "long": 5, "year": 5, "integr": 5, "contributor": 5, "dev": 5, "branch": 5, "coverag": 5, "compon": [5, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "refactor": 5, "conclus": 5, "appendix": 5, "special": 6, "case": 6, "mixtur": 6, "expert": 6, "error": 6, "rate": 6, "tutori": 7, "where": 7, "To": 7, "demo": [7, 70, 71], "transform": [8, 71], "len": [8, 70, 71], "api": 8, "content": 8, "transformer_len": [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69], "submodul": [9, 17, 42, 64], "subpackag": [9, 41], "activationcach": 10, "factoredmatrix": 11, "hookedencod": 12, "hookedencoderdecod": 13, "hookedtransform": 14, "hookedtransformerconfig": 15, "svdinterpret": 16, "abstract_attent": 18, "attent": [19, 67, 70], "bert_block": 20, "bert_emb": 21, "bert_mlm_head": 22, "emb": 23, "grouped_query_attent": 24, "layer_norm": 25, "layer_norm_pr": 26, "pos_emb": 27, "rms_norm": 28, "rms_norm_pr": 29, "t5_attent": 30, "t5_block": 31, "token_typed_emb": 32, "transformer_block": 33, "unemb": 34, "eval": 35, "head_detector": 36, "hook_point": 37, "loading_from_pretrain": 38, "past_key_value_cach": 39, "patch": [40, 70, 71], "pretrain": [41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62], "weight_convers": [42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62], "bert": 43, "bloom": 44, "coder": 45, "gemma": 46, "gpt2": 47, "gptj": 48, "llama": 49, "mingpt": 50, "mistral": 51, "mixtral": 52, "nanogpt": 53, "neel_solu_old": 54, "neo": 55, "neox": 56, "opt": 57, "phi": 58, "phi3": 59, "qwen": 60, "qwen2": 61, "t5": 62, "train": [63, 71], "util": [64, 65, 66, 67, 68, 69], "activation_funct": 65, "addmm": 66, "devic": 68, "exploratori": 70, "analysi": 70, "tip": 70, "thi": 70, "environ": 70, "ignor": 70, "import": [70, 71], "pytorch": 70, "plot": 70, "helper": 70, "indirect": [70, 71], "object": [70, 71], "identif": [70, 71], "brainstorm": 70, "what": 70, "": 70, "actual": 70, "go": 70, "On": 70, "option": 70, "direct": 70, "logit": 70, "attribut": 70, "layer": 70, "head": [70, 71], "activ": [70, 71], "residu": 70, "stream": 70, "decompos": 70, "consolid": 70, "understand": 70, "visual": 70, "pattern": 70, "compar": 70, "paper": 70, "bonu": 70, "explor": 70, "anomali": 70, "earli": 70, "ar": 70, "induct": [70, 71], "implic": 70, "backup": 70, "name": [70, 71], "mover": 70, "main": 71, "notebook": 71, "load": 71, "cach": 71, "all": 71, "hook": 71, "interven": 71, "task": 71, "avail": 71, "overview": 71, "open": 71, "sourc": 71, "librari": [71, 73], "some": 71, "friendli": 71, "i": 71, "ve": 71, "includ": 71, "resourc": 71, "architectur": 71, "paramet": 71, "fold": 71, "layernorm": 71, "For": 71, "curiou": 71, "featur": 71, "deal": 71, "token": 71, "gotcha": 71, "prepend_bo": 71, "factor": 71, "matrix": 71, "basic": 71, "exampl": 71, "medium": 71, "eigenvalu": 71, "copi": 71, "score": 71, "gener": [71, 73], "text": 71, "point": 71, "toi": 71, "pre": 71, "checkpoint": 71, "phase": 71, "transit": 71, "tabl": 72, "A": 73, "languag": 73}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 8, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "nbsphinx": 4, "sphinx": 57}, "alltitles": {"Citation": [[0, "citation"]], "Contributing": [[1, "contributing"]], "Setup": [[1, "setup"], [70, "Setup"], [71, "Setup"]], "DevContainer": [[1, "devcontainer"]], "Manual Setup": [[1, "manual-setup"]], "Testing": [[1, "testing"]], "Running the tests": [[1, "running-the-tests"]], "Formatting": [[1, "formatting"]], "Documentation": [[1, "documentation"]], "Docstring Style Guide": [[1, "docstring-style-guide"]], "Sections and Order": [[1, "sections-and-order"]], "Supported Sphinx Properties": [[1, "supported-sphinx-properties"]], "References to Other Functions/Classes": [[1, "references-to-other-functions-classes"]], "Maths": [[1, "maths"]], "Markup": [[1, "markup"]], "Gallery": [[2, "gallery"]], "Getting Started": [[3, "getting-started"]], "Advice for Reading the Code": [[3, "advice-for-reading-the-code"]], "Installation": [[3, "installation"]], "Huggingface Gated Access": [[3, "huggingface-gated-access"]], "Getting Started in Mechanistic Interpretability": [[4, "getting-started-in-mechanistic-interpretability"]], "TransformerLens 2.0": [[5, "transformerlens-2-0"]], "First, an introduction": [[5, "first-an-introduction"]], "Adopting Semantic Versioning": [[5, "adopting-semantic-versioning"]], "Deprecations": [[5, "deprecations"]], "Roadmap": [[5, "roadmap"]], "Immediate - within the next month": [[5, "immediate-within-the-next-month"]], "Mid-term - within the next 3 months": [[5, "mid-term-within-the-next-3-months"]], "Performance": [[5, "performance"]], "Streamlining Adding New Models": [[5, "streamlining-adding-new-models"]], "Long-term - within the next year": [[5, "long-term-within-the-next-year"]], "Model Testing": [[5, "model-testing"]], "Model Integration": [[5, "model-integration"]], "Contributors": [[5, "contributors"]], "New Dev Branches": [[5, "new-dev-branches"]], "Integration Tests": [[5, "integration-tests"]], "Test Coverage": [[5, "test-coverage"]], "Components Refactor": [[5, "components-refactor"]], "Conclusion": [[5, "conclusion"]], "Appendix": [[5, "appendix"]], "Semantic Versioning": [[5, "semantic-versioning"]], "Special Cases": [[6, "special-cases"]], "Mixture of Experts error rates": [[6, "mixture-of-experts-error-rates"]], "Tutorials": [[7, "tutorials"]], "Where To Start": [[7, "where-to-start"]], "Demos": [[7, "demos"]], "Transformer Lens API": [[8, "transformer-lens-api"]], "Contents": [[8, "contents"]], "transformer_lens": [[9, "transformer-lens"]], "Submodules": [[9, "submodules"], [17, "submodules"], [42, "submodules"], [64, "submodules"]], "Subpackages": [[9, "subpackages"], [41, "subpackages"]], "transformer_lens.ActivationCache": [[10, "module-transformer_lens.ActivationCache"]], "transformer_lens.FactoredMatrix": [[11, "module-transformer_lens.FactoredMatrix"]], "transformer_lens.HookedEncoder": [[12, "module-transformer_lens.HookedEncoder"]], "transformer_lens.HookedEncoderDecoder": [[13, "module-transformer_lens.HookedEncoderDecoder"]], "transformer_lens.HookedTransformer": [[14, "module-transformer_lens.HookedTransformer"]], "transformer_lens.HookedTransformerConfig": [[15, "module-transformer_lens.HookedTransformerConfig"]], "transformer_lens.SVDInterpreter": [[16, "module-transformer_lens.SVDInterpreter"]], "transformer_lens.components": [[17, "transformer-lens-components"]], "transformer_lens.components.abstract_attention": [[18, "module-transformer_lens.components.abstract_attention"]], "transformer_lens.components.attention": [[19, "module-transformer_lens.components.attention"]], "transformer_lens.components.bert_block": [[20, "module-transformer_lens.components.bert_block"]], "transformer_lens.components.bert_embed": [[21, "module-transformer_lens.components.bert_embed"]], "transformer_lens.components.bert_mlm_head": [[22, "module-transformer_lens.components.bert_mlm_head"]], "transformer_lens.components.embed": [[23, "module-transformer_lens.components.embed"]], "transformer_lens.components.grouped_query_attention": [[24, "module-transformer_lens.components.grouped_query_attention"]], "transformer_lens.components.layer_norm": [[25, "module-transformer_lens.components.layer_norm"]], "transformer_lens.components.layer_norm_pre": [[26, "module-transformer_lens.components.layer_norm_pre"]], "transformer_lens.components.pos_embed": [[27, "module-transformer_lens.components.pos_embed"]], "transformer_lens.components.rms_norm": [[28, "module-transformer_lens.components.rms_norm"]], "transformer_lens.components.rms_norm_pre": [[29, "module-transformer_lens.components.rms_norm_pre"]], "transformer_lens.components.t5_attention": [[30, "module-transformer_lens.components.t5_attention"]], "transformer_lens.components.t5_block": [[31, "module-transformer_lens.components.t5_block"]], "transformer_lens.components.token_typed_embed": [[32, "module-transformer_lens.components.token_typed_embed"]], "transformer_lens.components.transformer_block": [[33, "module-transformer_lens.components.transformer_block"]], "transformer_lens.components.unembed": [[34, "module-transformer_lens.components.unembed"]], "transformer_lens.evals": [[35, "module-transformer_lens.evals"]], "transformer_lens.head_detector": [[36, "module-transformer_lens.head_detector"]], "transformer_lens.hook_points": [[37, "module-transformer_lens.hook_points"]], "transformer_lens.loading_from_pretrained": [[38, "module-transformer_lens.loading_from_pretrained"]], "transformer_lens.past_key_value_caching": [[39, "module-transformer_lens.past_key_value_caching"]], "transformer_lens.patching": [[40, "module-transformer_lens.patching"]], "transformer_lens.pretrained": [[41, "transformer-lens-pretrained"]], "transformer_lens.pretrained.weight_conversions": [[42, "transformer-lens-pretrained-weight-conversions"]], "transformer_lens.pretrained.weight_conversions.bert": [[43, "module-transformer_lens.pretrained.weight_conversions.bert"]], "transformer_lens.pretrained.weight_conversions.bloom": [[44, "module-transformer_lens.pretrained.weight_conversions.bloom"]], "transformer_lens.pretrained.weight_conversions.coder": [[45, "module-transformer_lens.pretrained.weight_conversions.coder"]], "transformer_lens.pretrained.weight_conversions.gemma": [[46, "module-transformer_lens.pretrained.weight_conversions.gemma"]], "transformer_lens.pretrained.weight_conversions.gpt2": [[47, "module-transformer_lens.pretrained.weight_conversions.gpt2"]], "transformer_lens.pretrained.weight_conversions.gptj": [[48, "module-transformer_lens.pretrained.weight_conversions.gptj"]], "transformer_lens.pretrained.weight_conversions.llama": [[49, "module-transformer_lens.pretrained.weight_conversions.llama"]], "transformer_lens.pretrained.weight_conversions.mingpt": [[50, "module-transformer_lens.pretrained.weight_conversions.mingpt"]], "transformer_lens.pretrained.weight_conversions.mistral": [[51, "module-transformer_lens.pretrained.weight_conversions.mistral"]], "transformer_lens.pretrained.weight_conversions.mixtral": [[52, "module-transformer_lens.pretrained.weight_conversions.mixtral"]], "transformer_lens.pretrained.weight_conversions.nanogpt": [[53, "module-transformer_lens.pretrained.weight_conversions.nanogpt"]], "transformer_lens.pretrained.weight_conversions.neel_solu_old": [[54, "module-transformer_lens.pretrained.weight_conversions.neel_solu_old"]], "transformer_lens.pretrained.weight_conversions.neo": [[55, "module-transformer_lens.pretrained.weight_conversions.neo"]], "transformer_lens.pretrained.weight_conversions.neox": [[56, "module-transformer_lens.pretrained.weight_conversions.neox"]], "transformer_lens.pretrained.weight_conversions.opt": [[57, "module-transformer_lens.pretrained.weight_conversions.opt"]], "transformer_lens.pretrained.weight_conversions.phi": [[58, "module-transformer_lens.pretrained.weight_conversions.phi"]], "transformer_lens.pretrained.weight_conversions.phi3": [[59, "module-transformer_lens.pretrained.weight_conversions.phi3"]], "transformer_lens.pretrained.weight_conversions.qwen": [[60, "module-transformer_lens.pretrained.weight_conversions.qwen"]], "transformer_lens.pretrained.weight_conversions.qwen2": [[61, "module-transformer_lens.pretrained.weight_conversions.qwen2"]], "transformer_lens.pretrained.weight_conversions.t5": [[62, "module-transformer_lens.pretrained.weight_conversions.t5"]], "transformer_lens.train": [[63, "module-transformer_lens.train"]], "transformer_lens.utilities": [[64, "transformer-lens-utilities"]], "transformer_lens.utilities.activation_functions": [[65, "module-transformer_lens.utilities.activation_functions"]], "transformer_lens.utilities.addmm": [[66, "module-transformer_lens.utilities.addmm"]], "transformer_lens.utilities.attention": [[67, "module-transformer_lens.utilities.attention"]], "transformer_lens.utilities.devices": [[68, "module-transformer_lens.utilities.devices"]], "transformer_lens.utils": [[69, "module-transformer_lens.utils"]], "Exploratory Analysis Demo": [[70, "Exploratory-Analysis-Demo"]], "Tips for Reading This": [[70, "Tips-for-Reading-This"]], "Environment Setup (ignore)": [[70, "Environment-Setup-(ignore)"]], "Imports": [[70, "Imports"]], "PyTorch Setup": [[70, "PyTorch-Setup"]], "Plotting Helper Functions (ignore)": [[70, "Plotting-Helper-Functions-(ignore)"]], "Introduction": [[70, "Introduction"], [71, "Introduction"]], "Indirect Object Identification": [[70, "Indirect-Object-Identification"]], "Brainstorm What\u2019s Actually Going On (Optional)": [[70, "Brainstorm-What's-Actually-Going-On-(Optional)"]], "Direct Logit Attribution": [[70, "Direct-Logit-Attribution"]], "Logit Lens": [[70, "Logit-Lens"]], "Layer Attribution": [[70, "Layer-Attribution"]], "Head Attribution": [[70, "Head-Attribution"]], "Attention Analysis": [[70, "Attention-Analysis"]], "Activation Patching": [[70, "Activation-Patching"]], "Residual Stream": [[70, "Residual-Stream"]], "Layers": [[70, "Layers"]], "Heads": [[70, "Heads"]], "Decomposing Heads": [[70, "Decomposing-Heads"]], "Consolidating Understanding": [[70, "Consolidating-Understanding"]], "Visualizing Attention Patterns": [[70, "Visualizing-Attention-Patterns"]], "Comparing to the Paper": [[70, "Comparing-to-the-Paper"]], "Bonus: Exploring Anomalies": [[70, "Bonus:-Exploring-Anomalies"]], "Early Heads are Induction Heads(?!)": [[70, "Early-Heads-are-Induction-Heads(?!)"]], "Implications": [[70, "Implications"]], "Backup Name Mover Heads": [[70, "Backup-Name-Mover-Heads"]], "Transformer Lens Main Demo Notebook": [[71, "Transformer-Lens-Main-Demo-Notebook"]], "Loading and Running Models": [[71, "Loading-and-Running-Models"]], "Caching all Activations": [[71, "Caching-all-Activations"]], "Hooks: Intervening on Activations": [[71, "Hooks:-Intervening-on-Activations"]], "Activation Patching on the Indirect Object Identification Task": [[71, "Activation-Patching-on-the-Indirect-Object-Identification-Task"]], "Hooks: Accessing Activations": [[71, "Hooks:-Accessing-Activations"]], "Available Models": [[71, "Available-Models"]], "An overview of the important open source models in the library": [[71, "An-overview-of-the-important-open-source-models-in-the-library"]], "An overview of some interpretability-friendly models I\u2019ve trained and included": [[71, "An-overview-of-some-interpretability-friendly-models-I've-trained-and-included"]], "Other Resources:": [[71, "Other-Resources:"]], "Transformer architecture": [[71, "Transformer-architecture"]], "Parameter Names": [[71, "Parameter-Names"]], "Activation + Hook Names": [[71, "Activation-+-Hook-Names"]], "Folding LayerNorm (For the Curious)": [[71, "Folding-LayerNorm-(For-the-Curious)"]], "Features": [[71, "Features"]], "Dealing with tokens": [[71, "Dealing-with-tokens"]], "Gotcha: prepend_bos": [[71, "Gotcha:-prepend_bos"]], "Factored Matrix Class": [[71, "Factored-Matrix-Class"]], "Basic Examples": [[71, "Basic-Examples"]], "Medium Example: Eigenvalue Copying Scores": [[71, "Medium-Example:-Eigenvalue-Copying-Scores"]], "Generating Text": [[71, "Generating-Text"]], "Hook Points": [[71, "Hook-Points"]], "Toy Example": [[71, "Toy-Example"]], "Loading Pre-Trained Checkpoints": [[71, "Loading-Pre-Trained-Checkpoints"]], "Example: Induction Head Phase Transition": [[71, "Example:-Induction-Head-Phase-Transition"]], "Model Properties Table": [[72, "model-properties-table"]], "TransformerLens": [[73, "transformerlens"]], "A Library for Mechanistic Interpretability of Generative Language Models": [[73, "a-library-for-mechanistic-interpretability-of-generative-language-models"]]}, "indexentries": {"activationcache (class in transformer_lens.activationcache)": [[10, "transformer_lens.ActivationCache.ActivationCache"]], "accumulated_resid() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.accumulated_resid"]], "apply_ln_to_stack() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.apply_ln_to_stack"]], "apply_slice_to_batch_dim() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.apply_slice_to_batch_dim"]], "compute_head_results() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.compute_head_results"]], "decompose_resid() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.decompose_resid"]], "get_full_resid_decomposition() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.get_full_resid_decomposition"]], "get_neuron_results() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.get_neuron_results"]], "items() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.items"]], "keys() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.keys"]], "logit_attrs() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.logit_attrs"]], "module": [[10, "module-transformer_lens.ActivationCache"], [11, "module-transformer_lens.FactoredMatrix"], [12, "module-transformer_lens.HookedEncoder"], [13, "module-transformer_lens.HookedEncoderDecoder"], [14, "module-transformer_lens.HookedTransformer"], [15, "module-transformer_lens.HookedTransformerConfig"], [16, "module-transformer_lens.SVDInterpreter"], [18, "module-transformer_lens.components.abstract_attention"], [19, "module-transformer_lens.components.attention"], [20, "module-transformer_lens.components.bert_block"], [21, "module-transformer_lens.components.bert_embed"], [22, "module-transformer_lens.components.bert_mlm_head"], [23, "module-transformer_lens.components.embed"], [24, "module-transformer_lens.components.grouped_query_attention"], [25, "module-transformer_lens.components.layer_norm"], [26, "module-transformer_lens.components.layer_norm_pre"], [27, "module-transformer_lens.components.pos_embed"], [28, "module-transformer_lens.components.rms_norm"], [29, "module-transformer_lens.components.rms_norm_pre"], [30, "module-transformer_lens.components.t5_attention"], [31, "module-transformer_lens.components.t5_block"], [32, "module-transformer_lens.components.token_typed_embed"], [33, "module-transformer_lens.components.transformer_block"], [34, "module-transformer_lens.components.unembed"], [35, "module-transformer_lens.evals"], [36, "module-transformer_lens.head_detector"], [37, "module-transformer_lens.hook_points"], [38, "module-transformer_lens.loading_from_pretrained"], [39, "module-transformer_lens.past_key_value_caching"], [40, "module-transformer_lens.patching"], [43, "module-transformer_lens.pretrained.weight_conversions.bert"], [44, "module-transformer_lens.pretrained.weight_conversions.bloom"], [45, "module-transformer_lens.pretrained.weight_conversions.coder"], [46, "module-transformer_lens.pretrained.weight_conversions.gemma"], [47, "module-transformer_lens.pretrained.weight_conversions.gpt2"], [48, "module-transformer_lens.pretrained.weight_conversions.gptj"], [49, "module-transformer_lens.pretrained.weight_conversions.llama"], [50, "module-transformer_lens.pretrained.weight_conversions.mingpt"], [51, "module-transformer_lens.pretrained.weight_conversions.mistral"], [52, "module-transformer_lens.pretrained.weight_conversions.mixtral"], [53, "module-transformer_lens.pretrained.weight_conversions.nanogpt"], [54, "module-transformer_lens.pretrained.weight_conversions.neel_solu_old"], [55, "module-transformer_lens.pretrained.weight_conversions.neo"], [56, "module-transformer_lens.pretrained.weight_conversions.neox"], [57, "module-transformer_lens.pretrained.weight_conversions.opt"], [58, "module-transformer_lens.pretrained.weight_conversions.phi"], [59, "module-transformer_lens.pretrained.weight_conversions.phi3"], [60, "module-transformer_lens.pretrained.weight_conversions.qwen"], [61, "module-transformer_lens.pretrained.weight_conversions.qwen2"], [62, "module-transformer_lens.pretrained.weight_conversions.t5"], [63, "module-transformer_lens.train"], [65, "module-transformer_lens.utilities.activation_functions"], [66, "module-transformer_lens.utilities.addmm"], [67, "module-transformer_lens.utilities.attention"], [68, "module-transformer_lens.utilities.devices"], [69, "module-transformer_lens.utils"]], "remove_batch_dim() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.remove_batch_dim"]], "stack_activation() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.stack_activation"]], "stack_head_results() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.stack_head_results"]], "stack_neuron_results() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.stack_neuron_results"]], "to() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.to"]], "toggle_autodiff() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.toggle_autodiff"]], "transformer_lens.activationcache": [[10, "module-transformer_lens.ActivationCache"]], "values() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.values"]], "ab (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.AB"]], "ba (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.BA"]], "factoredmatrix (class in transformer_lens.factoredmatrix)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix"]], "s (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.S"]], "t (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.T"]], "u (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.U"]], "vh (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.Vh"]], "collapse_l() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.collapse_l"]], "collapse_r() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.collapse_r"]], "eigenvalues (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.eigenvalues"]], "get_corner() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.get_corner"]], "make_even() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.make_even"]], "ndim (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.ndim"]], "norm() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.norm"]], "pair (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.pair"]], "svd() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.svd"]], "transformer_lens.factoredmatrix": [[11, "module-transformer_lens.FactoredMatrix"]], "unsqueeze() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.unsqueeze"]], "hookedencoder (class in transformer_lens.hookedencoder)": [[12, "transformer_lens.HookedEncoder.HookedEncoder"]], "ov (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.OV"]], "qk (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.QK"]], "w_e (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_E"]], "w_e_pos (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_E_pos"]], "w_k (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_K"]], "w_o (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_O"]], "w_q (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_Q"]], "w_u (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_U"]], "w_v (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_V"]], "w_in (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_in"]], "w_out (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_out"]], "w_pos (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_pos"]], "all_head_labels() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.all_head_labels"]], "b_k (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_K"]], "b_o (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_O"]], "b_q (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_Q"]], "b_u (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_U"]], "b_v (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_V"]], "b_in (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_in"]], "b_out (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_out"]], "cpu() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.cpu"]], "cuda() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.cuda"]], "forward() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.forward"]], "from_pretrained() (transformer_lens.hookedencoder.hookedencoder class method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.from_pretrained"]], "mps() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.mps"]], "run_with_cache() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.run_with_cache"]], "to() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.to"]], "transformer_lens.hookedencoder": [[12, "module-transformer_lens.HookedEncoder"]], "hookedencoderdecoder (class in transformer_lens.hookedencoderdecoder)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder"]], "ov (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.OV"]], "qk (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.QK"]], "w_e (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_E"]], "w_k (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_K"]], "w_o (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_O"]], "w_q (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_Q"]], "w_u (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_U"]], "w_v (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_V"]], "w_in (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_in"]], "w_out (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_out"]], "w_pos (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_pos"]], "all_head_labels() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.all_head_labels"]], "b_k (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_K"]], "b_o (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_O"]], "b_q (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_Q"]], "b_u (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_U"]], "b_v (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_V"]], "b_in (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_in"]], "b_out (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_out"]], "cpu() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.cpu"]], "cuda() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.cuda"]], "forward() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.forward"]], "from_pretrained() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder class method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.from_pretrained"]], "mps() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.mps"]], "run_with_cache() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.run_with_cache"]], "to() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.to"]], "transformer_lens.hookedencoderdecoder": [[13, "module-transformer_lens.HookedEncoderDecoder"]], "hookedtransformer (class in transformer_lens.hookedtransformer)": [[14, "transformer_lens.HookedTransformer.HookedTransformer"]], "ov (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.OV"]], "output (class in transformer_lens.hookedtransformer)": [[14, "transformer_lens.HookedTransformer.Output"]], "qk (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.QK"]], "w_e (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_E"]], "w_e_pos (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_E_pos"]], "w_k (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_K"]], "w_o (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_O"]], "w_q (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_Q"]], "w_u (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_U"]], "w_v (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_V"]], "w_gate (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_gate"]], "w_in (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_in"]], "w_out (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_out"]], "w_pos (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_pos"]], "__init__() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.__init__"]], "accumulated_bias() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.accumulated_bias"]], "all_composition_scores() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.all_composition_scores"]], "all_head_labels() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.all_head_labels"]], "b_k (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_K"]], "b_o (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_O"]], "b_q (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_Q"]], "b_u (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_U"]], "b_v (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_V"]], "b_in (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_in"]], "b_out (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_out"]], "center_unembed() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.center_unembed"]], "center_writing_weights() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.center_writing_weights"]], "check_hooks_to_add() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.check_hooks_to_add"]], "cpu() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.cpu"]], "cuda() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.cuda"]], "fold_layer_norm() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.fold_layer_norm"]], "fold_value_biases() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.fold_value_biases"]], "forward() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.forward"]], "from_pretrained() (transformer_lens.hookedtransformer.hookedtransformer class method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.from_pretrained"]], "from_pretrained_no_processing() (transformer_lens.hookedtransformer.hookedtransformer class method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.from_pretrained_no_processing"]], "generate() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.generate"]], "get_token_position() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.get_token_position"]], "init_weights() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.init_weights"]], "input_to_embed() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.input_to_embed"]], "ln_final (transformer_lens.hookedtransformer.hookedtransformer attribute)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.ln_final"]], "load_and_process_state_dict() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.load_and_process_state_dict"]], "load_sample_training_dataset() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.load_sample_training_dataset"]], "logits (transformer_lens.hookedtransformer.output attribute)": [[14, "transformer_lens.HookedTransformer.Output.logits"]], "loss (transformer_lens.hookedtransformer.output attribute)": [[14, "transformer_lens.HookedTransformer.Output.loss"]], "loss_fn() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.loss_fn"]], "move_model_modules_to_device() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.move_model_modules_to_device"]], "mps() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.mps"]], "process_weights_() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.process_weights_"]], "refactor_factored_attn_matrices() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.refactor_factored_attn_matrices"]], "run_with_cache() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.run_with_cache"]], "sample_datapoint() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.sample_datapoint"]], "set_tokenizer() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.set_tokenizer"]], "set_ungroup_grouped_query_attention() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.set_ungroup_grouped_query_attention"]], "set_use_attn_in() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.set_use_attn_in"]], "set_use_attn_result() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.set_use_attn_result"]], "set_use_hook_mlp_in() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.set_use_hook_mlp_in"]], "set_use_split_qkv_input() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.set_use_split_qkv_input"]], "to() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.to"]], "to_single_str_token() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.to_single_str_token"]], "to_single_token() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.to_single_token"]], "to_str_tokens() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.to_str_tokens"]], "to_string() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.to_string"]], "to_tokens() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.to_tokens"]], "tokens_to_residual_directions() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.tokens_to_residual_directions"]], "transformer_lens.hookedtransformer": [[14, "module-transformer_lens.HookedTransformer"]], "hookedtransformerconfig (class in transformer_lens.hookedtransformerconfig)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig"]], "ntk_by_parts_factor (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.NTK_by_parts_factor"]], "ntk_by_parts_high_freq_factor (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.NTK_by_parts_high_freq_factor"]], "ntk_by_parts_low_freq_factor (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.NTK_by_parts_low_freq_factor"]], "act_fn (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.act_fn"]], "attention_dir (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.attention_dir"]], "attn_only (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.attn_only"]], "attn_scale (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.attn_scale"]], "attn_scores_soft_cap (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.attn_scores_soft_cap"]], "attn_types (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.attn_types"]], "checkpoint_index (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.checkpoint_index"]], "checkpoint_label_type (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.checkpoint_label_type"]], "checkpoint_value (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.checkpoint_value"]], "d_head (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.d_head"]], "d_mlp (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.d_mlp"]], "d_model (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.d_model"]], "d_vocab (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.d_vocab"]], "d_vocab_out (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.d_vocab_out"]], "decoder_start_token_id (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.decoder_start_token_id"]], "default_prepend_bos (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.default_prepend_bos"]], "device (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.device"]], "dtype (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.dtype"]], "eps (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.eps"]], "experts_per_token (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.experts_per_token"]], "final_rms (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.final_rms"]], "from_checkpoint (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.from_checkpoint"]], "from_dict() (transformer_lens.hookedtransformerconfig.hookedtransformerconfig class method)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.from_dict"]], "gated_mlp (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.gated_mlp"]], "init_mode (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.init_mode"]], "init_weights (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.init_weights"]], "initializer_range (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.initializer_range"]], "is_layer_norm_activation() (transformer_lens.hookedtransformerconfig.hookedtransformerconfig method)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.is_layer_norm_activation"]], "load_in_4bit (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.load_in_4bit"]], "model_name (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.model_name"]], "n_ctx (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.n_ctx"]], "n_devices (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.n_devices"]], "n_heads (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.n_heads"]], "n_key_value_heads (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.n_key_value_heads"]], "n_layers (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.n_layers"]], "n_params (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.n_params"]], "normalization_type (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.normalization_type"]], "num_experts (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.num_experts"]], "original_architecture (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.original_architecture"]], "output_logits_soft_cap (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.output_logits_soft_cap"]], "parallel_attn_mlp (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.parallel_attn_mlp"]], "positional_embedding_type (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.positional_embedding_type"]], "post_embedding_ln (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.post_embedding_ln"]], "relative_attention_max_distance (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.relative_attention_max_distance"]], "relative_attention_num_buckets (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.relative_attention_num_buckets"]], "rotary_adjacent_pairs (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.rotary_adjacent_pairs"]], "rotary_base (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.rotary_base"]], "rotary_dim (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.rotary_dim"]], "scale_attn_by_inverse_layer_idx (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.scale_attn_by_inverse_layer_idx"]], "seed (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.seed"]], "set_seed_everywhere() (transformer_lens.hookedtransformerconfig.hookedtransformerconfig method)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.set_seed_everywhere"]], "tie_word_embeddings (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.tie_word_embeddings"]], "to_dict() (transformer_lens.hookedtransformerconfig.hookedtransformerconfig method)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.to_dict"]], "tokenizer_name (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.tokenizer_name"]], "tokenizer_prepends_bos (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.tokenizer_prepends_bos"]], "transformer_lens.hookedtransformerconfig": [[15, "module-transformer_lens.HookedTransformerConfig"]], "trust_remote_code (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.trust_remote_code"]], "ungroup_grouped_query_attention (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.ungroup_grouped_query_attention"]], "unwrap() (transformer_lens.hookedtransformerconfig.hookedtransformerconfig class method)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.unwrap"]], "use_ntk_by_parts_rope (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_NTK_by_parts_rope"]], "use_attn_in (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_attn_in"]], "use_attn_result (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_attn_result"]], "use_attn_scale (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_attn_scale"]], "use_hook_mlp_in (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_hook_mlp_in"]], "use_hook_tokens (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_hook_tokens"]], "use_local_attn (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_local_attn"]], "use_normalization_before_and_after (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_normalization_before_and_after"]], "use_split_qkv_input (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_split_qkv_input"]], "window_size (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.window_size"]], "svdinterpreter (class in transformer_lens.svdinterpreter)": [[16, "transformer_lens.SVDInterpreter.SVDInterpreter"]], "get_singular_vectors() (transformer_lens.svdinterpreter.svdinterpreter method)": [[16, "transformer_lens.SVDInterpreter.SVDInterpreter.get_singular_vectors"]], "transformer_lens.svdinterpreter": [[16, "module-transformer_lens.SVDInterpreter"]], "abstractattention (class in transformer_lens.components.abstract_attention)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention"]], "ov (transformer_lens.components.abstract_attention.abstractattention property)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.OV"]], "qk (transformer_lens.components.abstract_attention.abstractattention property)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.QK"]], "__init__() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.__init__"]], "alibi (transformer_lens.components.abstract_attention.abstractattention attribute)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.alibi"]], "apply_causal_mask() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.apply_causal_mask"]], "apply_rotary() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.apply_rotary"]], "calculate_attention_scores() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.calculate_attention_scores"]], "calculate_qkv_matrices() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.calculate_qkv_matrices"]], "calculate_sin_cos_rotary() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.calculate_sin_cos_rotary"]], "calculate_z_scores() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.calculate_z_scores"]], "create_alibi_bias() (transformer_lens.components.abstract_attention.abstractattention static method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.create_alibi_bias"]], "create_alibi_multipliers() (transformer_lens.components.abstract_attention.abstractattention static method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.create_alibi_multipliers"]], "create_alibi_slope() (transformer_lens.components.abstract_attention.abstractattention static method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.create_alibi_slope"]], "forward() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.forward"]], "rotate_every_two() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.rotate_every_two"]], "transformer_lens.components.abstract_attention": [[18, "module-transformer_lens.components.abstract_attention"]], "attention (class in transformer_lens.components.attention)": [[19, "transformer_lens.components.attention.Attention"]], "__init__() (transformer_lens.components.attention.attention method)": [[19, "transformer_lens.components.attention.Attention.__init__"]], "transformer_lens.components.attention": [[19, "module-transformer_lens.components.attention"]], "bertblock (class in transformer_lens.components.bert_block)": [[20, "transformer_lens.components.bert_block.BertBlock"]], "forward() (transformer_lens.components.bert_block.bertblock method)": [[20, "transformer_lens.components.bert_block.BertBlock.forward"]], "transformer_lens.components.bert_block": [[20, "module-transformer_lens.components.bert_block"]], "bertembed (class in transformer_lens.components.bert_embed)": [[21, "transformer_lens.components.bert_embed.BertEmbed"]], "forward() (transformer_lens.components.bert_embed.bertembed method)": [[21, "transformer_lens.components.bert_embed.BertEmbed.forward"]], "transformer_lens.components.bert_embed": [[21, "module-transformer_lens.components.bert_embed"]], "bertmlmhead (class in transformer_lens.components.bert_mlm_head)": [[22, "transformer_lens.components.bert_mlm_head.BertMLMHead"]], "forward() (transformer_lens.components.bert_mlm_head.bertmlmhead method)": [[22, "transformer_lens.components.bert_mlm_head.BertMLMHead.forward"]], "transformer_lens.components.bert_mlm_head": [[22, "module-transformer_lens.components.bert_mlm_head"]], "embed (class in transformer_lens.components.embed)": [[23, "transformer_lens.components.embed.Embed"]], "forward() (transformer_lens.components.embed.embed method)": [[23, "transformer_lens.components.embed.Embed.forward"]], "transformer_lens.components.embed": [[23, "module-transformer_lens.components.embed"]], "groupedqueryattention (class in transformer_lens.components.grouped_query_attention)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention"]], "w_k (transformer_lens.components.grouped_query_attention.groupedqueryattention property)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.W_K"]], "w_v (transformer_lens.components.grouped_query_attention.groupedqueryattention property)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.W_V"]], "__init__() (transformer_lens.components.grouped_query_attention.groupedqueryattention method)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.__init__"]], "b_k (transformer_lens.components.grouped_query_attention.groupedqueryattention property)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.b_K"]], "b_v (transformer_lens.components.grouped_query_attention.groupedqueryattention property)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.b_V"]], "calculate_attention_scores() (transformer_lens.components.grouped_query_attention.groupedqueryattention method)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.calculate_attention_scores"]], "calculate_qkv_matrices() (transformer_lens.components.grouped_query_attention.groupedqueryattention method)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.calculate_qkv_matrices"]], "calculate_z_scores() (transformer_lens.components.grouped_query_attention.groupedqueryattention method)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.calculate_z_scores"]], "transformer_lens.components.grouped_query_attention": [[24, "module-transformer_lens.components.grouped_query_attention"]], "layernorm (class in transformer_lens.components.layer_norm)": [[25, "transformer_lens.components.layer_norm.LayerNorm"]], "__init__() (transformer_lens.components.layer_norm.layernorm method)": [[25, "transformer_lens.components.layer_norm.LayerNorm.__init__"]], "forward() (transformer_lens.components.layer_norm.layernorm method)": [[25, "transformer_lens.components.layer_norm.LayerNorm.forward"]], "transformer_lens.components.layer_norm": [[25, "module-transformer_lens.components.layer_norm"]], "layernormpre (class in transformer_lens.components.layer_norm_pre)": [[26, "transformer_lens.components.layer_norm_pre.LayerNormPre"]], "__init__() (transformer_lens.components.layer_norm_pre.layernormpre method)": [[26, "transformer_lens.components.layer_norm_pre.LayerNormPre.__init__"]], "forward() (transformer_lens.components.layer_norm_pre.layernormpre method)": [[26, "transformer_lens.components.layer_norm_pre.LayerNormPre.forward"]], "transformer_lens.components.layer_norm_pre": [[26, "module-transformer_lens.components.layer_norm_pre"]], "posembed (class in transformer_lens.components.pos_embed)": [[27, "transformer_lens.components.pos_embed.PosEmbed"]], "forward() (transformer_lens.components.pos_embed.posembed method)": [[27, "transformer_lens.components.pos_embed.PosEmbed.forward"]], "transformer_lens.components.pos_embed": [[27, "module-transformer_lens.components.pos_embed"]], "rmsnorm (class in transformer_lens.components.rms_norm)": [[28, "transformer_lens.components.rms_norm.RMSNorm"]], "__init__() (transformer_lens.components.rms_norm.rmsnorm method)": [[28, "transformer_lens.components.rms_norm.RMSNorm.__init__"]], "forward() (transformer_lens.components.rms_norm.rmsnorm method)": [[28, "transformer_lens.components.rms_norm.RMSNorm.forward"]], "transformer_lens.components.rms_norm": [[28, "module-transformer_lens.components.rms_norm"]], "rmsnormpre (class in transformer_lens.components.rms_norm_pre)": [[29, "transformer_lens.components.rms_norm_pre.RMSNormPre"]], "__init__() (transformer_lens.components.rms_norm_pre.rmsnormpre method)": [[29, "transformer_lens.components.rms_norm_pre.RMSNormPre.__init__"]], "forward() (transformer_lens.components.rms_norm_pre.rmsnormpre method)": [[29, "transformer_lens.components.rms_norm_pre.RMSNormPre.forward"]], "transformer_lens.components.rms_norm_pre": [[29, "module-transformer_lens.components.rms_norm_pre"]], "t5attention (class in transformer_lens.components.t5_attention)": [[30, "transformer_lens.components.t5_attention.T5Attention"]], "compute_relative_attention_bias() (transformer_lens.components.t5_attention.t5attention method)": [[30, "transformer_lens.components.t5_attention.T5Attention.compute_relative_attention_bias"]], "transformer_lens.components.t5_attention": [[30, "module-transformer_lens.components.t5_attention"]], "t5block (class in transformer_lens.components.t5_block)": [[31, "transformer_lens.components.t5_block.T5Block"]], "forward() (transformer_lens.components.t5_block.t5block method)": [[31, "transformer_lens.components.t5_block.T5Block.forward"]], "transformer_lens.components.t5_block": [[31, "module-transformer_lens.components.t5_block"]], "tokentypeembed (class in transformer_lens.components.token_typed_embed)": [[32, "transformer_lens.components.token_typed_embed.TokenTypeEmbed"]], "forward() (transformer_lens.components.token_typed_embed.tokentypeembed method)": [[32, "transformer_lens.components.token_typed_embed.TokenTypeEmbed.forward"]], "transformer_lens.components.token_typed_embed": [[32, "module-transformer_lens.components.token_typed_embed"]], "transformerblock (class in transformer_lens.components.transformer_block)": [[33, "transformer_lens.components.transformer_block.TransformerBlock"]], "apply_mlp() (transformer_lens.components.transformer_block.transformerblock method)": [[33, "transformer_lens.components.transformer_block.TransformerBlock.apply_mlp"]], "forward() (transformer_lens.components.transformer_block.transformerblock method)": [[33, "transformer_lens.components.transformer_block.TransformerBlock.forward"]], "ln1 (transformer_lens.components.transformer_block.transformerblock attribute)": [[33, "transformer_lens.components.transformer_block.TransformerBlock.ln1"]], "ln2 (transformer_lens.components.transformer_block.transformerblock attribute)": [[33, "transformer_lens.components.transformer_block.TransformerBlock.ln2"]], "mlp (transformer_lens.components.transformer_block.transformerblock attribute)": [[33, "transformer_lens.components.transformer_block.TransformerBlock.mlp"]], "transformer_lens.components.transformer_block": [[33, "module-transformer_lens.components.transformer_block"]], "unembed (class in transformer_lens.components.unembed)": [[34, "transformer_lens.components.unembed.Unembed"]], "forward() (transformer_lens.components.unembed.unembed method)": [[34, "transformer_lens.components.unembed.Unembed.forward"]], "transformer_lens.components.unembed": [[34, "module-transformer_lens.components.unembed"]], "ioidataset (class in transformer_lens.evals)": [[35, "transformer_lens.evals.IOIDataset"]], "evaluate() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.evaluate"]], "evaluate_on_dataset() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.evaluate_on_dataset"]], "get_default_names() (transformer_lens.evals.ioidataset static method)": [[35, "transformer_lens.evals.IOIDataset.get_default_names"]], "get_default_nouns() (transformer_lens.evals.ioidataset static method)": [[35, "transformer_lens.evals.IOIDataset.get_default_nouns"]], "get_default_templates() (transformer_lens.evals.ioidataset static method)": [[35, "transformer_lens.evals.IOIDataset.get_default_templates"]], "get_sample() (transformer_lens.evals.ioidataset method)": [[35, "transformer_lens.evals.IOIDataset.get_sample"]], "induction_loss() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.induction_loss"]], "ioi_eval() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.ioi_eval"]], "make_code_data_loader() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.make_code_data_loader"]], "make_owt_data_loader() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.make_owt_data_loader"]], "make_pile_data_loader() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.make_pile_data_loader"]], "make_wiki_data_loader() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.make_wiki_data_loader"]], "sanity_check() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.sanity_check"]], "transformer_lens.evals": [[35, "module-transformer_lens.evals"]], "compute_head_attention_similarity_score() (in module transformer_lens.head_detector)": [[36, "transformer_lens.head_detector.compute_head_attention_similarity_score"]], "detect_head() (in module transformer_lens.head_detector)": [[36, "transformer_lens.head_detector.detect_head"]], "get_duplicate_token_head_detection_pattern() (in module transformer_lens.head_detector)": [[36, "transformer_lens.head_detector.get_duplicate_token_head_detection_pattern"]], "get_induction_head_detection_pattern() (in module transformer_lens.head_detector)": [[36, "transformer_lens.head_detector.get_induction_head_detection_pattern"]], "get_previous_token_head_detection_pattern() (in module transformer_lens.head_detector)": [[36, "transformer_lens.head_detector.get_previous_token_head_detection_pattern"]], "get_supported_heads() (in module transformer_lens.head_detector)": [[36, "transformer_lens.head_detector.get_supported_heads"]], "transformer_lens.head_detector": [[36, "module-transformer_lens.head_detector"]], "hookfunction (in module transformer_lens.hook_points)": [[37, "transformer_lens.hook_points.HookFunction"]], "hookpoint (class in transformer_lens.hook_points)": [[37, "transformer_lens.hook_points.HookPoint"]], "hookedrootmodule (class in transformer_lens.hook_points)": [[37, "transformer_lens.hook_points.HookedRootModule"]], "lenshandle (class in transformer_lens.hook_points)": [[37, "transformer_lens.hook_points.LensHandle"]], "add_caching_hooks() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.add_caching_hooks"]], "add_hook() (transformer_lens.hook_points.hookpoint method)": [[37, "transformer_lens.hook_points.HookPoint.add_hook"]], "add_hook() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.add_hook"]], "add_perma_hook() (transformer_lens.hook_points.hookpoint method)": [[37, "transformer_lens.hook_points.HookPoint.add_perma_hook"]], "add_perma_hook() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.add_perma_hook"]], "cache_all() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.cache_all"]], "cache_some() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.cache_some"]], "check_and_add_hook() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.check_and_add_hook"]], "check_hooks_to_add() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.check_hooks_to_add"]], "clear_context() (transformer_lens.hook_points.hookpoint method)": [[37, "transformer_lens.hook_points.HookPoint.clear_context"]], "clear_contexts() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.clear_contexts"]], "context_level (transformer_lens.hook_points.lenshandle attribute)": [[37, "transformer_lens.hook_points.LensHandle.context_level"]], "forward() (transformer_lens.hook_points.hookpoint method)": [[37, "transformer_lens.hook_points.HookPoint.forward"]], "get_caching_hooks() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.get_caching_hooks"]], "hook (transformer_lens.hook_points.lenshandle attribute)": [[37, "transformer_lens.hook_points.LensHandle.hook"]], "hook_dict (transformer_lens.hook_points.hookedrootmodule attribute)": [[37, "transformer_lens.hook_points.HookedRootModule.hook_dict"]], "hook_points() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.hook_points"]], "hooks() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.hooks"]], "is_permanent (transformer_lens.hook_points.lenshandle attribute)": [[37, "transformer_lens.hook_points.LensHandle.is_permanent"]], "layer() (transformer_lens.hook_points.hookpoint method)": [[37, "transformer_lens.hook_points.HookPoint.layer"]], "mod_dict (transformer_lens.hook_points.hookedrootmodule attribute)": [[37, "transformer_lens.hook_points.HookedRootModule.mod_dict"]], "name (transformer_lens.hook_points.hookedrootmodule attribute)": [[37, "transformer_lens.hook_points.HookedRootModule.name"]], "remove_all_hook_fns() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.remove_all_hook_fns"]], "remove_hooks() (transformer_lens.hook_points.hookpoint method)": [[37, "transformer_lens.hook_points.HookPoint.remove_hooks"]], "reset_hooks() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.reset_hooks"]], "run_with_cache() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.run_with_cache"]], "run_with_hooks() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.run_with_hooks"]], "setup() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.setup"]], "transformer_lens.hook_points": [[37, "module-transformer_lens.hook_points"]], "config (class in transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.Config"]], "model_aliases (in module transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.MODEL_ALIASES"]], "non_hf_hosted_model_names (in module transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.NON_HF_HOSTED_MODEL_NAMES"]], "official_model_names (in module transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.OFFICIAL_MODEL_NAMES"]], "d_head (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.d_head"]], "d_mlp (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.d_mlp"]], "d_model (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.d_model"]], "d_vocab (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.d_vocab"]], "debug (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.debug"]], "get_checkpoint_labels() (in module transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.get_checkpoint_labels"]], "get_num_params_of_pretrained() (in module transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.get_num_params_of_pretrained"]], "get_pretrained_model_config() (in module transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.get_pretrained_model_config"]], "init_range (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.init_range"]], "layer_norm_eps (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.layer_norm_eps"]], "n_ctx (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.n_ctx"]], "n_heads (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.n_heads"]], "n_layers (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.n_layers"]], "transformer_lens.loading_from_pretrained": [[38, "module-transformer_lens.loading_from_pretrained"]], "hookedtransformerkeyvaluecache (class in transformer_lens.past_key_value_caching)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache"]], "hookedtransformerkeyvaluecacheentry (class in transformer_lens.past_key_value_caching)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry"]], "append() (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecacheentry method)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry.append"]], "append_attention_mask() (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache method)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.append_attention_mask"]], "entries (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache attribute)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.entries"]], "freeze() (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache method)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.freeze"]], "frozen (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache attribute)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.frozen"]], "frozen (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecacheentry attribute)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry.frozen"]], "init_cache() (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache class method)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.init_cache"]], "init_cache_entry() (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecacheentry class method)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry.init_cache_entry"]], "past_keys (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecacheentry attribute)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry.past_keys"]], "past_values (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecacheentry attribute)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry.past_values"]], "previous_attention_mask (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache attribute)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.previous_attention_mask"]], "transformer_lens.past_key_value_caching": [[39, "module-transformer_lens.past_key_value_caching"]], "unfreeze() (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache method)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.unfreeze"]], "generic_activation_patch() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.generic_activation_patch"]], "get_act_patch_attn_head_all_pos_every() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_all_pos_every"]], "get_act_patch_attn_head_by_pos_every() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_by_pos_every"]], "get_act_patch_attn_head_k_all_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_k_all_pos"]], "get_act_patch_attn_head_k_by_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_k_by_pos"]], "get_act_patch_attn_head_out_all_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_out_all_pos"]], "get_act_patch_attn_head_out_by_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_out_by_pos"]], "get_act_patch_attn_head_pattern_all_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_pattern_all_pos"]], "get_act_patch_attn_head_pattern_by_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_pattern_by_pos"]], "get_act_patch_attn_head_pattern_dest_src_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_pattern_dest_src_pos"]], "get_act_patch_attn_head_q_all_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_q_all_pos"]], "get_act_patch_attn_head_q_by_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_q_by_pos"]], "get_act_patch_attn_head_v_all_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_v_all_pos"]], "get_act_patch_attn_head_v_by_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_v_by_pos"]], "get_act_patch_attn_out() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_out"]], "get_act_patch_block_every() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_block_every"]], "get_act_patch_mlp_out() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_mlp_out"]], "get_act_patch_resid_mid() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_resid_mid"]], "get_act_patch_resid_pre() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_resid_pre"]], "layer_head_dest_src_pos_pattern_patch_setter() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.layer_head_dest_src_pos_pattern_patch_setter"]], "layer_head_pattern_patch_setter() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.layer_head_pattern_patch_setter"]], "layer_head_pos_pattern_patch_setter() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.layer_head_pos_pattern_patch_setter"]], "layer_head_vector_patch_setter() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.layer_head_vector_patch_setter"]], "layer_pos_head_vector_patch_setter() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.layer_pos_head_vector_patch_setter"]], "layer_pos_patch_setter() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.layer_pos_patch_setter"]], "transformer_lens.patching": [[40, "module-transformer_lens.patching"]], "transformer_lens.pretrained.weight_conversions.bert": [[43, "module-transformer_lens.pretrained.weight_conversions.bert"]], "convert_bloom_weights() (in module transformer_lens.pretrained.weight_conversions.bloom)": [[44, "transformer_lens.pretrained.weight_conversions.bloom.convert_bloom_weights"]], "transformer_lens.pretrained.weight_conversions.bloom": [[44, "module-transformer_lens.pretrained.weight_conversions.bloom"]], "convert_coder_weights() (in module transformer_lens.pretrained.weight_conversions.coder)": [[45, "transformer_lens.pretrained.weight_conversions.coder.convert_coder_weights"]], "transformer_lens.pretrained.weight_conversions.coder": [[45, "module-transformer_lens.pretrained.weight_conversions.coder"]], "transformer_lens.pretrained.weight_conversions.gemma": [[46, "module-transformer_lens.pretrained.weight_conversions.gemma"]], "transformer_lens.pretrained.weight_conversions.gpt2": [[47, "module-transformer_lens.pretrained.weight_conversions.gpt2"]], "transformer_lens.pretrained.weight_conversions.gptj": [[48, "module-transformer_lens.pretrained.weight_conversions.gptj"]], "transformer_lens.pretrained.weight_conversions.llama": [[49, "module-transformer_lens.pretrained.weight_conversions.llama"]], "transformer_lens.pretrained.weight_conversions.mingpt": [[50, "module-transformer_lens.pretrained.weight_conversions.mingpt"]], "convert_mistral_weights() (in module transformer_lens.pretrained.weight_conversions.mistral)": [[51, "transformer_lens.pretrained.weight_conversions.mistral.convert_mistral_weights"]], "transformer_lens.pretrained.weight_conversions.mistral": [[51, "module-transformer_lens.pretrained.weight_conversions.mistral"]], "convert_mixtral_weights() (in module transformer_lens.pretrained.weight_conversions.mixtral)": [[52, "transformer_lens.pretrained.weight_conversions.mixtral.convert_mixtral_weights"]], "transformer_lens.pretrained.weight_conversions.mixtral": [[52, "module-transformer_lens.pretrained.weight_conversions.mixtral"]], "transformer_lens.pretrained.weight_conversions.nanogpt": [[53, "module-transformer_lens.pretrained.weight_conversions.nanogpt"]], "transformer_lens.pretrained.weight_conversions.neel_solu_old": [[54, "module-transformer_lens.pretrained.weight_conversions.neel_solu_old"]], "transformer_lens.pretrained.weight_conversions.neo": [[55, "module-transformer_lens.pretrained.weight_conversions.neo"]], "transformer_lens.pretrained.weight_conversions.neox": [[56, "module-transformer_lens.pretrained.weight_conversions.neox"]], "transformer_lens.pretrained.weight_conversions.opt": [[57, "module-transformer_lens.pretrained.weight_conversions.opt"]], "convert_phi_weights() (in module transformer_lens.pretrained.weight_conversions.phi)": [[58, "transformer_lens.pretrained.weight_conversions.phi.convert_phi_weights"]], "transformer_lens.pretrained.weight_conversions.phi": [[58, "module-transformer_lens.pretrained.weight_conversions.phi"]], "convert_phi3_weights() (in module transformer_lens.pretrained.weight_conversions.phi3)": [[59, "transformer_lens.pretrained.weight_conversions.phi3.convert_phi3_weights"]], "transformer_lens.pretrained.weight_conversions.phi3": [[59, "module-transformer_lens.pretrained.weight_conversions.phi3"]], "convert_qwen_weights() (in module transformer_lens.pretrained.weight_conversions.qwen)": [[60, "transformer_lens.pretrained.weight_conversions.qwen.convert_qwen_weights"]], "transformer_lens.pretrained.weight_conversions.qwen": [[60, "module-transformer_lens.pretrained.weight_conversions.qwen"]], "convert_qwen2_weights() (in module transformer_lens.pretrained.weight_conversions.qwen2)": [[61, "transformer_lens.pretrained.weight_conversions.qwen2.convert_qwen2_weights"]], "transformer_lens.pretrained.weight_conversions.qwen2": [[61, "module-transformer_lens.pretrained.weight_conversions.qwen2"]], "convert_t5_weights() (in module transformer_lens.pretrained.weight_conversions.t5)": [[62, "transformer_lens.pretrained.weight_conversions.t5.convert_t5_weights"]], "transformer_lens.pretrained.weight_conversions.t5": [[62, "module-transformer_lens.pretrained.weight_conversions.t5"]], "hookedtransformertrainconfig (class in transformer_lens.train)": [[63, "transformer_lens.train.HookedTransformerTrainConfig"]], "batch_size (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.batch_size"]], "device (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.device"]], "lr (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.lr"]], "max_grad_norm (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.max_grad_norm"]], "max_steps (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.max_steps"]], "momentum (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.momentum"]], "num_epochs (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.num_epochs"]], "optimizer_name (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.optimizer_name"]], "print_every (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.print_every"]], "save_dir (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.save_dir"]], "save_every (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.save_every"]], "seed (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.seed"]], "train() (in module transformer_lens.train)": [[63, "transformer_lens.train.train"]], "transformer_lens.train": [[63, "module-transformer_lens.train"]], "wandb (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.wandb"]], "wandb_project_name (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.wandb_project_name"]], "warmup_steps (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.warmup_steps"]], "weight_decay (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.weight_decay"]], "transformer_lens.utilities.activation_functions": [[65, "module-transformer_lens.utilities.activation_functions"]], "batch_addmm() (in module transformer_lens.utilities.addmm)": [[66, "transformer_lens.utilities.addmm.batch_addmm"]], "transformer_lens.utilities.addmm": [[66, "module-transformer_lens.utilities.addmm"]], "vanilla_addmm() (in module transformer_lens.utilities.addmm)": [[66, "transformer_lens.utilities.addmm.vanilla_addmm"]], "complex_attn_linear() (in module transformer_lens.utilities.attention)": [[67, "transformer_lens.utilities.attention.complex_attn_linear"]], "simple_attn_linear() (in module transformer_lens.utilities.attention)": [[67, "transformer_lens.utilities.attention.simple_attn_linear"]], "transformer_lens.utilities.attention": [[67, "module-transformer_lens.utilities.attention"]], "get_device_for_block_index() (in module transformer_lens.utilities.devices)": [[68, "transformer_lens.utilities.devices.get_device_for_block_index"]], "move_to_and_update_config() (in module transformer_lens.utilities.devices)": [[68, "transformer_lens.utilities.devices.move_to_and_update_config"]], "transformer_lens.utilities.devices": [[68, "module-transformer_lens.utilities.devices"]], "locallyoverridendefaults (class in transformer_lens.utils)": [[69, "transformer_lens.utils.LocallyOverridenDefaults"]], "slice (class in transformer_lens.utils)": [[69, "transformer_lens.utils.Slice"]], "sliceinput (in module transformer_lens.utils)": [[69, "transformer_lens.utils.SliceInput"]], "__init__() (transformer_lens.utils.locallyoverridendefaults method)": [[69, "transformer_lens.utils.LocallyOverridenDefaults.__init__"]], "__init__() (transformer_lens.utils.slice method)": [[69, "transformer_lens.utils.Slice.__init__"]], "apply() (transformer_lens.utils.slice method)": [[69, "transformer_lens.utils.Slice.apply"]], "calc_fan_in_and_fan_out() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.calc_fan_in_and_fan_out"]], "composition_scores() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.composition_scores"]], "download_file_from_hf() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.download_file_from_hf"]], "gelu_fast() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.gelu_fast"]], "gelu_new() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.gelu_new"]], "get_act_name() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_act_name"]], "get_attention_mask() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_attention_mask"]], "get_corner() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_corner"]], "get_cumsum_along_dim() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_cumsum_along_dim"]], "get_dataset() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_dataset"]], "get_device() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_device"]], "get_input_with_manually_prepended_bos() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_input_with_manually_prepended_bos"]], "get_nested_attr() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_nested_attr"]], "get_offset_position_ids() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_offset_position_ids"]], "get_tokenizer_with_bos() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_tokenizer_with_bos"]], "get_tokens_with_bos_removed() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_tokens_with_bos_removed"]], "indices() (transformer_lens.utils.slice method)": [[69, "transformer_lens.utils.Slice.indices"]], "init_kaiming_normal_() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.init_kaiming_normal_"]], "init_kaiming_uniform_() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.init_kaiming_uniform_"]], "init_xavier_normal_() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.init_xavier_normal_"]], "init_xavier_uniform_() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.init_xavier_uniform_"]], "is_lower_triangular() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.is_lower_triangular"]], "is_square() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.is_square"]], "keep_single_column() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.keep_single_column"]], "lm_accuracy() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.lm_accuracy"]], "lm_cross_entropy_loss() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.lm_cross_entropy_loss"]], "override_or_use_default_value() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.override_or_use_default_value"]], "print_gpu_mem() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.print_gpu_mem"]], "remove_batch_dim() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.remove_batch_dim"]], "repeat_along_head_dimension() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.repeat_along_head_dimension"]], "sample_logits() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.sample_logits"]], "set_nested_attr() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.set_nested_attr"]], "slice (transformer_lens.utils.slice attribute)": [[69, "transformer_lens.utils.Slice.slice"]], "solu() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.solu"]], "test_prompt() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.test_prompt"]], "to_numpy() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.to_numpy"]], "tokenize_and_concatenate() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.tokenize_and_concatenate"]], "transformer_lens.utils": [[69, "module-transformer_lens.utils"]], "transpose() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.transpose"]], "unwrap() (transformer_lens.utils.slice class method)": [[69, "transformer_lens.utils.Slice.unwrap"]]}})
\ No newline at end of file
+Search.setIndex({"docnames": ["content/citation", "content/contributing", "content/gallery", "content/getting_started", "content/getting_started_mech_interp", "content/news/release-2.0", "content/special_cases", "content/tutorials", "generated/code/modules", "generated/code/transformer_lens", "generated/code/transformer_lens.ActivationCache", "generated/code/transformer_lens.FactoredMatrix", "generated/code/transformer_lens.HookedEncoder", "generated/code/transformer_lens.HookedEncoderDecoder", "generated/code/transformer_lens.HookedTransformer", "generated/code/transformer_lens.HookedTransformerConfig", "generated/code/transformer_lens.SVDInterpreter", "generated/code/transformer_lens.components", "generated/code/transformer_lens.components.abstract_attention", "generated/code/transformer_lens.components.attention", "generated/code/transformer_lens.components.bert_block", "generated/code/transformer_lens.components.bert_embed", "generated/code/transformer_lens.components.bert_mlm_head", "generated/code/transformer_lens.components.embed", "generated/code/transformer_lens.components.grouped_query_attention", "generated/code/transformer_lens.components.layer_norm", "generated/code/transformer_lens.components.layer_norm_pre", "generated/code/transformer_lens.components.pos_embed", "generated/code/transformer_lens.components.rms_norm", "generated/code/transformer_lens.components.rms_norm_pre", "generated/code/transformer_lens.components.t5_attention", "generated/code/transformer_lens.components.t5_block", "generated/code/transformer_lens.components.token_typed_embed", "generated/code/transformer_lens.components.transformer_block", "generated/code/transformer_lens.components.unembed", "generated/code/transformer_lens.evals", "generated/code/transformer_lens.head_detector", "generated/code/transformer_lens.hook_points", "generated/code/transformer_lens.loading_from_pretrained", "generated/code/transformer_lens.past_key_value_caching", "generated/code/transformer_lens.patching", "generated/code/transformer_lens.pretrained", "generated/code/transformer_lens.pretrained.weight_conversions", "generated/code/transformer_lens.pretrained.weight_conversions.bert", "generated/code/transformer_lens.pretrained.weight_conversions.bloom", "generated/code/transformer_lens.pretrained.weight_conversions.coder", "generated/code/transformer_lens.pretrained.weight_conversions.gemma", "generated/code/transformer_lens.pretrained.weight_conversions.gpt2", "generated/code/transformer_lens.pretrained.weight_conversions.gptj", "generated/code/transformer_lens.pretrained.weight_conversions.llama", "generated/code/transformer_lens.pretrained.weight_conversions.mingpt", "generated/code/transformer_lens.pretrained.weight_conversions.mistral", "generated/code/transformer_lens.pretrained.weight_conversions.mixtral", "generated/code/transformer_lens.pretrained.weight_conversions.nanogpt", "generated/code/transformer_lens.pretrained.weight_conversions.neel_solu_old", "generated/code/transformer_lens.pretrained.weight_conversions.neo", "generated/code/transformer_lens.pretrained.weight_conversions.neox", "generated/code/transformer_lens.pretrained.weight_conversions.opt", "generated/code/transformer_lens.pretrained.weight_conversions.phi", "generated/code/transformer_lens.pretrained.weight_conversions.phi3", "generated/code/transformer_lens.pretrained.weight_conversions.qwen", "generated/code/transformer_lens.pretrained.weight_conversions.qwen2", "generated/code/transformer_lens.pretrained.weight_conversions.t5", "generated/code/transformer_lens.train", "generated/code/transformer_lens.utilities", "generated/code/transformer_lens.utilities.activation_functions", "generated/code/transformer_lens.utilities.addmm", "generated/code/transformer_lens.utilities.attention", "generated/code/transformer_lens.utilities.devices", "generated/code/transformer_lens.utils", "generated/demos/Exploratory_Analysis_Demo", "generated/demos/Main_Demo", "generated/model_properties_table", "index"], "filenames": ["content/citation.md", "content/contributing.md", "content/gallery.md", "content/getting_started.md", "content/getting_started_mech_interp.md", "content/news/release-2.0.md", "content/special_cases.md", "content/tutorials.md", "generated/code/modules.rst", "generated/code/transformer_lens.rst", "generated/code/transformer_lens.ActivationCache.rst", "generated/code/transformer_lens.FactoredMatrix.rst", "generated/code/transformer_lens.HookedEncoder.rst", "generated/code/transformer_lens.HookedEncoderDecoder.rst", "generated/code/transformer_lens.HookedTransformer.rst", "generated/code/transformer_lens.HookedTransformerConfig.rst", "generated/code/transformer_lens.SVDInterpreter.rst", "generated/code/transformer_lens.components.rst", "generated/code/transformer_lens.components.abstract_attention.rst", "generated/code/transformer_lens.components.attention.rst", "generated/code/transformer_lens.components.bert_block.rst", "generated/code/transformer_lens.components.bert_embed.rst", "generated/code/transformer_lens.components.bert_mlm_head.rst", "generated/code/transformer_lens.components.embed.rst", "generated/code/transformer_lens.components.grouped_query_attention.rst", "generated/code/transformer_lens.components.layer_norm.rst", "generated/code/transformer_lens.components.layer_norm_pre.rst", "generated/code/transformer_lens.components.pos_embed.rst", "generated/code/transformer_lens.components.rms_norm.rst", "generated/code/transformer_lens.components.rms_norm_pre.rst", "generated/code/transformer_lens.components.t5_attention.rst", "generated/code/transformer_lens.components.t5_block.rst", "generated/code/transformer_lens.components.token_typed_embed.rst", "generated/code/transformer_lens.components.transformer_block.rst", "generated/code/transformer_lens.components.unembed.rst", "generated/code/transformer_lens.evals.rst", "generated/code/transformer_lens.head_detector.rst", "generated/code/transformer_lens.hook_points.rst", "generated/code/transformer_lens.loading_from_pretrained.rst", "generated/code/transformer_lens.past_key_value_caching.rst", "generated/code/transformer_lens.patching.rst", "generated/code/transformer_lens.pretrained.rst", "generated/code/transformer_lens.pretrained.weight_conversions.rst", "generated/code/transformer_lens.pretrained.weight_conversions.bert.rst", "generated/code/transformer_lens.pretrained.weight_conversions.bloom.rst", "generated/code/transformer_lens.pretrained.weight_conversions.coder.rst", "generated/code/transformer_lens.pretrained.weight_conversions.gemma.rst", "generated/code/transformer_lens.pretrained.weight_conversions.gpt2.rst", "generated/code/transformer_lens.pretrained.weight_conversions.gptj.rst", "generated/code/transformer_lens.pretrained.weight_conversions.llama.rst", "generated/code/transformer_lens.pretrained.weight_conversions.mingpt.rst", "generated/code/transformer_lens.pretrained.weight_conversions.mistral.rst", "generated/code/transformer_lens.pretrained.weight_conversions.mixtral.rst", "generated/code/transformer_lens.pretrained.weight_conversions.nanogpt.rst", "generated/code/transformer_lens.pretrained.weight_conversions.neel_solu_old.rst", "generated/code/transformer_lens.pretrained.weight_conversions.neo.rst", "generated/code/transformer_lens.pretrained.weight_conversions.neox.rst", "generated/code/transformer_lens.pretrained.weight_conversions.opt.rst", "generated/code/transformer_lens.pretrained.weight_conversions.phi.rst", "generated/code/transformer_lens.pretrained.weight_conversions.phi3.rst", "generated/code/transformer_lens.pretrained.weight_conversions.qwen.rst", "generated/code/transformer_lens.pretrained.weight_conversions.qwen2.rst", "generated/code/transformer_lens.pretrained.weight_conversions.t5.rst", "generated/code/transformer_lens.train.rst", "generated/code/transformer_lens.utilities.rst", "generated/code/transformer_lens.utilities.activation_functions.rst", "generated/code/transformer_lens.utilities.addmm.rst", "generated/code/transformer_lens.utilities.attention.rst", "generated/code/transformer_lens.utilities.devices.rst", "generated/code/transformer_lens.utils.rst", "generated/demos/Exploratory_Analysis_Demo.ipynb", "generated/demos/Main_Demo.ipynb", "generated/model_properties_table.md", "index.md"], "titles": ["Citation", "Contributing", "Gallery", "Getting Started", "Getting Started in Mechanistic Interpretability", "TransformerLens 2.0", "Special Cases", "Tutorials", "Transformer Lens API", "transformer_lens", "transformer_lens.ActivationCache", "transformer_lens.FactoredMatrix", "transformer_lens.HookedEncoder", "transformer_lens.HookedEncoderDecoder", "transformer_lens.HookedTransformer", "transformer_lens.HookedTransformerConfig", "transformer_lens.SVDInterpreter", "transformer_lens.components", "transformer_lens.components.abstract_attention", "transformer_lens.components.attention", "transformer_lens.components.bert_block", "transformer_lens.components.bert_embed", "transformer_lens.components.bert_mlm_head", "transformer_lens.components.embed", "transformer_lens.components.grouped_query_attention", "transformer_lens.components.layer_norm", "transformer_lens.components.layer_norm_pre", "transformer_lens.components.pos_embed", "transformer_lens.components.rms_norm", "transformer_lens.components.rms_norm_pre", "transformer_lens.components.t5_attention", "transformer_lens.components.t5_block", "transformer_lens.components.token_typed_embed", "transformer_lens.components.transformer_block", "transformer_lens.components.unembed", "transformer_lens.evals", "transformer_lens.head_detector", "transformer_lens.hook_points", "transformer_lens.loading_from_pretrained", "transformer_lens.past_key_value_caching", "transformer_lens.patching", "transformer_lens.pretrained", "transformer_lens.pretrained.weight_conversions", "transformer_lens.pretrained.weight_conversions.bert", "transformer_lens.pretrained.weight_conversions.bloom", "transformer_lens.pretrained.weight_conversions.coder", "transformer_lens.pretrained.weight_conversions.gemma", "transformer_lens.pretrained.weight_conversions.gpt2", "transformer_lens.pretrained.weight_conversions.gptj", "transformer_lens.pretrained.weight_conversions.llama", "transformer_lens.pretrained.weight_conversions.mingpt", "transformer_lens.pretrained.weight_conversions.mistral", "transformer_lens.pretrained.weight_conversions.mixtral", "transformer_lens.pretrained.weight_conversions.nanogpt", "transformer_lens.pretrained.weight_conversions.neel_solu_old", "transformer_lens.pretrained.weight_conversions.neo", "transformer_lens.pretrained.weight_conversions.neox", "transformer_lens.pretrained.weight_conversions.opt", "transformer_lens.pretrained.weight_conversions.phi", "transformer_lens.pretrained.weight_conversions.phi3", "transformer_lens.pretrained.weight_conversions.qwen", "transformer_lens.pretrained.weight_conversions.qwen2", "transformer_lens.pretrained.weight_conversions.t5", "transformer_lens.train", "transformer_lens.utilities", "transformer_lens.utilities.activation_functions", "transformer_lens.utilities.addmm", "transformer_lens.utilities.attention", "transformer_lens.utilities.devices", "transformer_lens.utils", "Exploratory Analysis Demo", "Transformer Lens Main Demo Notebook", "Model Properties Table", "TransformerLens"], "terms": {"pleas": [0, 1, 3, 4, 5, 71], "cite": 0, "thi": [0, 1, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 32, 33, 34, 35, 36, 37, 38, 39, 40, 63, 67, 68, 69, 71, 73], "librari": [0, 2, 3, 4, 5, 7, 10, 35, 69, 70], "misc": 0, "nanda2022transformerlen": 0, "titl": [0, 1, 70, 71], "transformerlen": [0, 2, 3, 4, 7, 10, 14, 18, 38, 69, 70, 71], "author": [0, 70], "neel": [0, 2, 4, 7, 14, 16, 71], "nanda": [0, 2, 4, 14, 71], "joseph": [0, 5], "bloom": [0, 5, 14, 18, 38, 42, 72], "year": 0, "2022": [0, 69], "howpublish": 0, "url": [0, 3], "http": [0, 1, 3, 7, 10, 14, 15, 18, 24, 32, 35, 36, 40, 66, 69, 70, 71], "github": [0, 1, 3, 7, 14, 66], "com": [0, 3, 7, 10, 14, 18, 66, 70, 71], "transformerlensorg": [0, 3, 7], "For": [1, 5, 10, 12, 13, 14, 18, 32, 36, 69, 70], "one": [1, 3, 4, 5, 10, 12, 13, 14, 15, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 36, 37, 38, 39, 40, 69, 70, 71, 73], "click": [1, 71], "your": [1, 3, 5, 7, 14, 15, 36, 37, 70, 71], "develop": [1, 5, 7, 70, 71], "environ": [1, 3, 73], "project": [1, 5, 7, 10, 18, 24, 63, 70], "includ": [1, 4, 5, 7, 10, 12, 13, 14, 15, 35, 36, 37, 70], "It": [1, 3, 5, 7, 10, 12, 13, 14, 15, 18, 35, 37, 40, 69, 70, 71, 73], "can": [1, 2, 3, 5, 6, 7, 10, 11, 12, 13, 14, 15, 16, 18, 19, 24, 35, 36, 37, 38, 39, 40, 69, 70, 71, 73], "us": [1, 2, 3, 5, 6, 7, 10, 12, 13, 14, 15, 16, 18, 19, 24, 26, 31, 33, 35, 36, 37, 38, 39, 40, 63, 67, 68, 69, 70, 71, 73], "local": [1, 14, 15, 18, 19, 24, 38, 69, 71], "v": [1, 10, 12, 13, 14, 15, 18, 24, 40, 70, 71], "code": [1, 4, 5, 10, 14, 15, 18, 35, 36, 37, 38, 69, 70, 71], "codespac": 1, "poetri": 1, "packag": 1, "manag": [1, 5, 10, 14, 37, 69], "instal": [1, 5, 70, 71], "follow": [1, 3, 5, 10, 14, 69, 71, 73], "also": [1, 5, 7, 10, 12, 13, 14, 15, 16, 31, 36, 37, 38, 68, 69, 70, 71], "virtual": 1, "config": [1, 14, 15, 18, 19, 24, 38, 40, 63], "virtualenv": 1, "true": [1, 10, 12, 13, 14, 15, 30, 31, 35, 36, 37, 38, 40, 68, 69, 70, 71], "dev": 1, "doc": [1, 5, 8, 10, 14, 71], "jupyt": 1, "If": [1, 3, 5, 8, 10, 12, 13, 14, 15, 25, 28, 32, 36, 37, 38, 40, 68, 69, 70, 71], "ad": [1, 7, 14, 15, 18, 37, 70, 71], "featur": [1, 3, 5, 7, 12, 13, 16, 18, 40, 69, 70, 73], "add": [1, 5, 14, 15, 18, 37, 39, 66, 69, 70, 71, 73], "unit": [1, 5], "you": [1, 3, 4, 5, 7, 10, 12, 13, 14, 15, 16, 30, 35, 36, 37, 38, 69, 70, 71, 73], "need": [1, 3, 5, 10, 14, 15, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 69, 70, 71, 73], "model": [1, 2, 3, 6, 7, 10, 12, 13, 14, 15, 16, 18, 19, 21, 24, 35, 36, 37, 38, 40, 45, 63, 68, 69, 70], "ones": [1, 12, 13, 14, 31, 36, 70], "ar": [1, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15, 18, 20, 24, 35, 36, 37, 38, 39, 40, 69, 71, 73], "cach": [1, 10, 14, 15, 31, 33, 36, 37, 39, 40, 69, 70, 73], "action": [1, 2, 70], "so": [1, 5, 7, 10, 11, 12, 13, 14, 15, 16, 35, 37, 38, 39, 40, 69, 70, 71], "quickli": [1, 5, 7, 73], "cd": [1, 70, 71], "These": [1, 70, 71], "gpt2": [1, 14, 15, 16, 18, 19, 24, 35, 38, 42, 70, 71, 72], "attn": [1, 10, 12, 13, 14, 15, 18, 30, 38, 40, 69, 70, 71, 72], "onli": [1, 2, 5, 10, 11, 12, 13, 14, 15, 18, 19, 24, 26, 31, 33, 36, 37, 38, 69, 70, 71, 72], "1l": [1, 38, 70, 71, 72], "2l": [1, 14, 38, 71, 72], "3l": [1, 38, 71, 72], "4l": [1, 38, 71, 72], "tini": [1, 10, 14, 38, 69, 70, 71, 72], "stori": [1, 10, 14, 38, 40, 69, 70, 72], "1m": [1, 10, 14, 38, 69, 72], "note": [1, 3, 5, 10, 11, 12, 13, 14, 15, 18, 24, 35, 37, 38, 66, 69, 70, 71], "i": [1, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15, 16, 18, 22, 24, 26, 31, 32, 33, 35, 36, 37, 38, 39, 40, 63, 67, 68, 69, 70, 73], "quit": [1, 5], "slow": [1, 71], "we": [1, 2, 5, 8, 10, 14, 15, 36, 39, 40, 69, 70, 71, 73], "have": [1, 3, 5, 10, 12, 13, 14, 18, 19, 24, 36, 40, 69, 70, 71, 73], "cpu": [1, 10, 12, 13, 14, 15, 18, 38, 70, 71], "smaller": [1, 5, 71], "like": [1, 3, 4, 5, 7, 12, 13, 14, 15, 21, 30, 35, 36, 40, 69, 70, 71, 73], "prefer": 1, "possibl": [1, 5, 12, 13, 14, 36, 40, 69, 70, 71, 73], "via": [1, 2, 3, 4, 5, 12, 13, 14, 40, 70], "make": [1, 3, 5, 7, 11, 12, 13, 14, 36, 37, 70, 71, 73], "accept": [1, 3, 5, 12, 13, 14, 37, 70], "notebook": [1, 3, 7, 70, 73], "all": [1, 4, 5, 10, 12, 13, 14, 15, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 32, 33, 34, 35, 36, 37, 40, 65, 69, 70, 73], "suit": 1, "mention": [1, 5, 71], "pycln": 1, "isort": 1, "black": [1, 71], "pull": [1, 5], "request": [1, 5], "check": [1, 3, 5, 7, 14, 16, 18, 35, 36, 37, 69, 70, 71], "file": [1, 5, 69], "line": [1, 5, 70, 71], "length": [1, 10, 12, 13, 14, 15, 18, 25, 26, 27, 28, 29, 69, 70, 71], "set": [1, 2, 5, 10, 12, 13, 14, 15, 18, 35, 36, 37, 40, 63, 69, 70, 71], "100": [1, 35, 70, 71], "pyproject": 1, "toml": 1, "instead": [1, 6, 10, 14, 15, 18, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 70, 71], "default": [1, 6, 10, 14, 15, 16, 18, 19, 24, 27, 31, 33, 35, 36, 37, 38, 40, 69, 70, 71], "88": [1, 71], "sure": [1, 3, 5, 14, 70, 71], "thorough": 1, "ani": [1, 3, 10, 12, 13, 14, 15, 18, 19, 24, 37, 69, 70, 71, 73], "should": [1, 5, 7, 10, 12, 13, 14, 18, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 36, 37, 63, 69, 70, 71], "do": [1, 3, 4, 5, 7, 10, 12, 13, 14, 18, 35, 37, 40, 69, 70, 71, 73], "directli": [1, 5, 12, 13, 15, 69, 70, 71], "automat": [1, 5, 7, 14, 15, 69, 70, 71], "gener": [1, 5, 7, 13, 14, 18, 31, 33, 35, 39, 40, 69, 70], "api": [1, 5, 37, 70], "when": [1, 3, 5, 7, 10, 11, 12, 13, 14, 15, 24, 31, 33, 35, 37, 38, 39, 40, 67, 69, 70, 71], "merg": [1, 5, 14], "main": [1, 3, 5, 6, 7, 10, 37, 70], "thei": [1, 4, 5, 14, 15, 18, 35, 40, 69, 70, 71, 73], "pytest": 1, "doctest": 1, "want": [1, 5, 7, 10, 14, 16, 35, 36, 37, 39, 69, 70, 71], "view": [1, 2], "chang": [1, 2, 3, 5, 14, 15, 37, 40, 69, 70, 71], "hot": [1, 70, 71], "reload": [1, 70, 71], "give": [1, 5, 10, 14, 15, 35, 38, 40, 69, 70, 71, 73], "real": [1, 7, 69, 70, 71, 73], "time": [1, 5, 7, 8, 10, 14, 36, 37, 69, 70, 71], "edit": [1, 7, 14, 40, 70, 71, 73], "googl": [1, 7, 38, 70, 71], "python": [1, 2, 15, 30, 35, 38, 69, 71, 72], "write": [1, 2, 3, 5, 14, 69, 70, 71, 73], "some": [1, 3, 5, 10, 14, 16, 18, 35, 37, 40, 69, 70], "from": [1, 2, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15, 16, 18, 24, 32, 35, 36, 38, 40, 69, 70, 71, 73], "restructuredtext": 1, "rest": [1, 5, 10, 14, 15, 35, 38, 69, 71], "In": [1, 2, 5, 6, 10, 12, 13, 14, 35, 37, 70, 71], "case": [1, 2, 5, 10, 14, 15, 35, 36, 37, 38, 40, 69, 70, 71, 72], "A": [1, 2, 4, 5, 10, 11, 12, 13, 14, 18, 24, 31, 32, 33, 35, 37, 39, 40, 69, 70, 71], "descript": 1, "what": [1, 3, 5, 7, 10, 14, 36, 37, 40, 71, 73], "doe": [1, 5, 10, 12, 13, 14, 15, 36, 37, 40, 69, 70, 71], "much": [1, 5, 10, 14, 35, 36, 40, 69, 70, 71], "detail": [1, 5, 10, 14, 15, 18, 24, 33, 37, 38, 40, 69, 70, 71], "necessari": [1, 5, 71], "fulli": [1, 40, 70], "understand": [1, 10, 14, 36, 71], "warn": [1, 10, 14, 36, 37, 69], "user": [1, 2, 5, 14, 15, 38, 69, 71], "e": [1, 5, 10, 12, 13, 14, 15, 18, 36, 37, 38, 69, 70, 71, 72], "g": [1, 5, 10, 12, 13, 14, 36, 37, 38, 69, 71], "common": [1, 5, 7, 10, 14, 15, 18, 69, 70, 71], "pitfal": 1, "exampl": [1, 2, 10, 12, 13, 14, 16, 18, 32, 35, 37, 69, 70], "here": [1, 2, 3, 5, 14, 15, 18, 19, 24, 35, 36, 69, 70, 71], "print": [1, 10, 35, 63, 69, 70, 71], "1": [1, 3, 4, 5, 10, 12, 13, 14, 15, 18, 19, 20, 24, 30, 31, 32, 36, 37, 38, 39, 69, 70, 71, 72], "2": [1, 3, 4, 10, 12, 13, 14, 15, 18, 35, 36, 38, 69, 70, 71, 72, 73], "3": [1, 6, 10, 11, 12, 13, 14, 15, 18, 35, 38, 40, 69, 70, 71, 72, 73], "arg": [1, 24, 37], "param_without_type_signatur": 1, "each": [1, 5, 10, 11, 12, 13, 14, 15, 18, 36, 37, 38, 39, 40, 67, 69, 70, 71], "indent": 1, "onc": [1, 3, 5, 14, 69, 70, 71], "more": [1, 5, 7, 10, 11, 14, 15, 18, 32, 36, 40, 69, 70, 71, 73], "param_2": 1, "anoth": [1, 5, 70, 71, 73], "paramet": [1, 5, 7, 10, 12, 13, 14, 15, 16, 18, 19, 24, 25, 26, 27, 31, 33, 35, 36, 37, 38, 40, 68, 69, 70], "return": [1, 5, 10, 11, 12, 13, 14, 18, 24, 27, 31, 33, 35, 36, 37, 38, 40, 63, 68, 69, 70, 71], "without": [1, 3, 5, 10, 14, 28, 29, 69, 70, 71], "type": [1, 6, 7, 10, 12, 13, 14, 15, 16, 21, 24, 27, 31, 32, 33, 36, 37, 38, 40, 63, 68, 69, 70, 71], "signatur": [1, 12, 13, 14, 71], "rais": [1, 14, 36, 38, 69, 71], "inform": [1, 14, 15, 32, 37, 38, 70, 71], "about": [1, 5, 7, 10, 14, 35, 37, 40, 69, 70, 71, 73], "error": [1, 10, 14, 36, 38, 71], "mai": [1, 5, 10, 12, 13, 14, 15, 18, 69, 70, 71], "part": [1, 5, 10, 14, 15, 26, 40, 70, 71, 73], "codebas": [1, 71], "cross": [1, 10, 14, 30, 31, 69, 70, 71], "referenc": [1, 5], "omit": [1, 36, 71], "full": [1, 4, 5, 10, 12, 14, 15, 18, 69, 71], "path": [1, 4, 69], "same": [1, 3, 5, 10, 11, 14, 15, 18, 36, 37, 39, 40, 67, 69, 70, 71], "mod": 1, "transformer_len": [1, 3, 5, 8, 70, 71], "modul": [1, 5, 8, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 23, 25, 26, 27, 28, 29, 31, 32, 33, 34, 35, 37, 38, 39, 40, 69, 71], "const": 1, "loading_from_pretrain": [1, 8, 9, 14, 71], "official_model_nam": [1, 14, 38], "hookedtransform": [1, 3, 5, 6, 7, 8, 9, 10, 12, 13, 15, 16, 35, 36, 38, 40, 63, 68, 69, 70, 71], "meth": [1, 10], "from_pretrain": [1, 3, 6, 10, 12, 13, 14, 16, 35, 38, 69, 70, 71], "attr": 1, "cfg": [1, 12, 13, 14, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 39, 44, 45, 51, 52, 58, 59, 60, 61, 62, 68, 69, 70, 71], "latex": 1, "re": [1, 3, 4, 7, 10, 14, 15, 36, 70, 71], "place": [1, 5, 7, 12, 13, 14, 69, 70, 71, 73], "string": [1, 12, 13, 14, 15, 35, 36, 37, 38, 69, 70, 71], "backward": [1, 37, 70, 71], "slash": 1, "must": [1, 10, 12, 13, 14, 15, 18, 36, 37, 66, 69, 70, 71], "repeat": [1, 14, 35, 39, 69, 70, 71], "inlin": 1, "displai": [1, 70, 71], "mode": [1, 10, 14, 15, 18, 26, 69, 70, 71], "b": [1, 11, 12, 14, 32, 35, 38, 40, 67, 69, 71, 72], "2ab": 1, "nowrap": 1, "begin": [1, 3, 5, 13, 14, 35, 36, 69, 70, 71], "eqnarrai": 1, "y": [1, 10, 14, 36, 70, 71], "ax": [1, 14, 40, 71], "bx": 1, "c": [1, 38, 69, 71, 72], "f": [1, 70, 71], "x": [1, 5, 10, 14, 18, 25, 26, 28, 29, 36, 37, 38, 66, 69, 70, 71], "2xy": 1, "end": [1, 5, 7, 14, 37, 40, 69, 70, 71], "ital": 1, "text": [1, 7, 10, 14, 15, 18, 31, 33, 35, 37, 38, 39, 69, 70], "bold": 1, "list": [1, 3, 4, 10, 12, 13, 14, 15, 35, 36, 37, 39, 40, 69, 70, 71], "item": [1, 5, 10, 69, 70, 71], "number": [1, 5, 10, 14, 15, 16, 18, 35, 38, 40, 63, 68, 69, 70, 71], "quot": 1, "level": [1, 37, 70, 71, 73], "extern": [1, 70], "link": [1, 14, 35], "domain": 1, "invalid": 1, "research": [2, 3, 4, 5, 7, 70, 71, 73], "done": [2, 4, 5, 6, 10, 14, 15, 18, 37, 70, 71], "involv": [2, 5, 70, 71], "progress": [2, 5, 14, 71], "measur": [2, 35, 36, 40, 69, 70], "grokk": [2, 7], "mechanist": [2, 3, 7, 40, 70, 71], "interpret": [2, 3, 7, 10, 14, 16, 36, 40, 69, 70], "iclr": 2, "spotlight": 2, "2023": 2, "lawrenc": 2, "chan": 2, "tom": [2, 70], "lieberum": 2, "jess": 2, "smith": 2, "jacob": 2, "steinhardt": 2, "find": [2, 5, 7, 10, 11, 14, 40, 70, 71], "neuron": [2, 7, 10, 14, 70, 71], "haystack": 2, "studi": [2, 4, 40, 70, 71], "spars": [2, 5], "probe": 2, "gurne": 2, "matthew": 2, "pauli": 2, "katherin": 2, "harvei": 2, "dmitrii": 2, "troitskii": 2, "dimitri": 2, "bertsima": 2, "toward": [2, 18, 40, 70], "autom": 2, "circuit": [2, 12, 14, 18, 35, 36, 40, 69, 70, 71], "discoveri": 2, "arthur": [2, 71], "conmi": [2, 71], "augustin": 2, "n": [2, 14, 18, 63, 66, 69, 70, 71], "mavor": 2, "parker": 2, "aengu": 2, "lynch": 2, "stefan": 2, "heimersheim": 2, "adri\u00e0": 2, "garriga": 2, "alonso": 2, "actual": [2, 5, 14, 36, 37, 71], "othello": [2, 7, 38, 72], "gpt": [2, 3, 4, 7, 10, 12, 13, 14, 15, 18, 19, 24, 35, 38, 69, 70, 71, 72, 73], "ha": [2, 3, 4, 5, 10, 11, 12, 13, 14, 18, 24, 35, 38, 39, 40, 67, 69, 70, 71], "linear": [2, 7, 12, 13, 14, 18, 38, 67, 70, 71], "emerg": [2, 7], "world": [2, 5, 7, 71, 73], "represent": [2, 7], "docstr": 2, "4": [2, 3, 5, 15, 18, 35, 69, 70, 71, 72], "layer": [2, 6, 10, 12, 13, 14, 15, 16, 18, 19, 21, 24, 25, 26, 36, 37, 38, 39, 40, 67, 68, 69, 71], "attent": [2, 7, 10, 12, 13, 14, 15, 17, 18, 20, 24, 27, 30, 31, 33, 36, 40, 64, 69, 71], "transform": [2, 3, 4, 7, 10, 12, 13, 14, 15, 18, 19, 20, 21, 22, 23, 25, 26, 27, 28, 29, 31, 32, 33, 34, 38, 39, 40, 66, 69, 70], "jett": 2, "janiak": 2, "toi": [2, 14], "univers": 2, "icml": 2, "bilal": 2, "chughtai": 2, "n2g": 2, "scalabl": 2, "approach": [2, 5, 10, 70, 71], "quantifi": [2, 36], "larg": [2, 5, 7, 15, 18, 38, 69, 71, 72, 73], "languag": [2, 12, 14, 35, 63, 69, 70, 71], "workshop": 2, "rtml": 2, "alex": [2, 71], "foot": [2, 14, 71], "esben": 2, "kran": 2, "ioanni": 2, "konsta": 2, "fazl": 2, "barez": 2, "elicit": 2, "latent": 2, "predict": [2, 7, 10, 12, 14, 22, 35, 36, 69, 70, 71], "tune": [2, 12, 13, 38, 69, 71, 72], "len": [2, 10, 38], "nora": 2, "belros": 2, "zach": 2, "furman": 2, "logan": 2, "danni": 2, "halawi": 2, "igor": 2, "ostrovski": 2, "lev": 2, "mckinnei": 2, "stella": 2, "biderman": 2, "contribut": [2, 5, 10, 14, 70], "being": [2, 5, 10, 12, 13, 14, 15, 36, 37, 40, 69, 70, 71], "induct": [2, 4, 35, 36, 38], "head": [2, 4, 7, 10, 12, 13, 14, 15, 16, 18, 22, 24, 35, 36, 38, 40, 67, 69], "phase": 2, "replic": [2, 4, 14, 16, 35, 70, 71], "partial": [2, 70, 71], "context": [2, 10, 14, 37, 40, 69, 70, 71], "learn": [2, 3, 7, 15, 63, 69, 70, 71, 73], "connor": 2, "kissan": 2, "decis": [2, 3], "script": [2, 7], "train": [2, 7, 8, 9, 10, 12, 13, 14, 15, 35, 38, 69, 70, 73], "which": [2, 3, 5, 6, 7, 10, 12, 13, 14, 15, 35, 36, 37, 38, 39, 40, 69, 70, 71, 73], "intermedi": [2, 10, 14, 37, 71], "activ": [2, 3, 4, 5, 7, 10, 12, 13, 14, 15, 16, 37, 40, 65, 69, 73], "perform": [2, 6, 7, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 35, 37, 69, 70, 71], "attribut": [2, 4, 7, 10, 18, 40, 69, 71], "ablat": [2, 70, 71], "up": [2, 3, 4, 5, 10, 14, 15, 37, 40, 63, 69, 70, 71], "initi": [2, 5, 8, 14, 15, 24, 37, 68, 69, 70, 71], "work": [2, 3, 4, 5, 7, 10, 12, 13, 14, 18, 37, 38, 69, 70, 71, 73], "found": [2, 3, 5, 6, 14, 15, 70, 71], "demo": [3, 5, 16, 38, 72, 73], "how": [3, 5, 7, 10, 14, 36, 40, 63, 70, 71, 73], "basic": [3, 7, 14, 35, 69, 70], "To": [3, 4, 5, 10, 14, 15, 18, 37, 70, 71], "see": [3, 5, 7, 10, 12, 13, 14, 15, 18, 24, 32, 33, 36, 37, 38, 40, 69, 70, 71, 73], "exploratori": [3, 7, 36, 69, 71, 73], "analysi": [3, 7, 10, 14, 36, 69, 71, 73], "practic": [3, 4, 7, 70, 71], "look": [3, 4, 5, 7, 8, 10, 14, 18, 36, 40, 69, 70, 71, 73], "out": [3, 5, 7, 10, 14, 16, 40, 69, 70, 71], "my": [3, 5, 7, 14, 15, 69, 70, 71, 73], "analys": [3, 7, 10, 14, 71], "indirect": [3, 4, 7, 35], "object": [3, 4, 5, 7, 10, 11, 12, 13, 14, 15, 16, 35, 37, 38, 39, 63, 69], "identif": [3, 4, 7, 35], "record": [3, 7, 71], "myself": [3, 5, 7, 71], "veri": [3, 4, 5, 7, 10, 15, 16, 35, 70, 71, 73], "young": [3, 4, 71], "small": [3, 4, 5, 6, 7, 10, 14, 15, 35, 38, 69, 70, 71, 72, 73], "field": [3, 4, 5, 14, 69, 71, 73], "lot": [3, 4, 5, 7, 10, 11, 39, 40, 69, 70, 71, 73], "open": [3, 4, 5, 14, 35, 73], "problem": [3, 4, 5, 71, 73], "would": [3, 4, 5, 12, 18, 32, 70, 71, 73], "help": [3, 4, 5, 15, 40, 70, 71, 73], "try": [3, 4, 10, 14, 36, 70, 71], "concret": [3, 4, 70, 71], "figur": [3, 40, 70, 71], "where": [3, 5, 6, 10, 11, 12, 13, 14, 15, 18, 33, 36, 37, 38, 40, 63, 69, 70, 71], "skill": [3, 71], "kei": [3, 4, 5, 10, 12, 13, 14, 15, 18, 19, 24, 30, 31, 33, 35, 36, 39, 40, 69, 70, 71], "resourc": [3, 4, 5], "new": [3, 7, 10, 14, 37, 38, 39, 69, 70, 71], "tutori": [3, 4, 5, 70, 71], "scratch": [3, 4, 70], "an": [3, 4, 7, 10, 11, 12, 13, 14, 15, 18, 35, 36, 37, 38, 39, 40, 63, 67, 69, 70, 73], "accompani": [3, 4, 7, 71], "templat": [3, 35], "yourself": [3, 14, 70, 71], "One": [3, 5, 14, 70, 71, 73], "signific": [3, 37, 70, 71], "design": [3, 5, 10, 70, 71, 73], "made": [3, 5, 35, 70, 71], "wa": [3, 5, 6, 10, 12, 13, 14, 15, 35, 40, 70, 71], "singl": [3, 5, 10, 12, 14, 18, 31, 32, 33, 39, 40, 69, 70, 71], "implement": [3, 5, 12, 13, 14, 18, 40, 66, 69, 70, 71], "could": [3, 5, 70, 71], "support": [3, 5, 7, 12, 13, 14, 15, 30, 36, 37, 65, 66, 69, 70, 71], "rang": [3, 4, 5, 14, 16, 36, 40, 69, 70, 71], "subtli": [3, 18], "differ": [3, 5, 6, 10, 12, 13, 14, 15, 18, 35, 36, 37, 40, 69, 70, 71], "style": [3, 5, 10, 12, 13, 14, 15, 19, 36, 70, 71, 73], "upsid": 3, "just": [3, 4, 5, 10, 14, 15, 35, 40, 69, 70, 71], "arbitrari": [3, 14, 70, 71], "name": [3, 5, 10, 14, 15, 35, 36, 37, 38, 40, 63, 69], "But": [3, 10, 14, 40, 69, 70, 71], "downsid": 3, "py": [3, 5, 12, 13, 66], "compon": [3, 8, 9, 10, 12, 13, 14, 15, 67, 69, 70, 71], "difficult": [3, 10], "recommend": [3, 8, 10, 14, 15, 16, 37, 70, 71], "clean": [3, 40, 69, 70, 71], "minim": [3, 5, 71], "intern": [3, 5, 10, 14, 40, 70, 71, 73], "architectur": [3, 12, 13, 70], "significantli": [3, 12, 13, 14, 35, 40, 70, 71], "clearer": 3, "better": [3, 14, 15, 35, 36, 38, 70, 71], "document": [3, 14, 69, 71], "pip": [3, 5, 70, 71], "git": 3, "import": [3, 5, 10, 14, 16, 35, 39, 40, 69, 73], "known": [3, 73], "easytransform": [3, 71, 73], "break": [3, 5, 10, 70, 71], "been": [3, 5, 10, 14, 69, 71], "sinc": [3, 5, 10, 14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 70, 71], "renam": [3, 5], "old": [3, 5, 38, 71], "version": [3, 7, 14, 35, 37, 66, 70, 71], "legaci": [3, 36], "run": [3, 5, 6, 10, 14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 38, 39, 40, 63, 70, 73], "v1": 3, "avail": [3, 5, 7, 10, 14, 15, 36, 38], "requir": [3, 5, 12, 13, 14, 40, 69, 71], "luckili": 3, "provid": [3, 10, 12, 13, 14, 25, 28, 32, 37, 68, 69], "wai": [3, 5, 6, 10, 14, 15, 37, 69, 70, 71], "those": [3, 5, 6, 14, 37, 69, 70], "configur": [3, 5, 15, 63, 68], "environment": 3, "variabl": [3, 12, 13], "simpli": [3, 5, 70], "token": [3, 7, 10, 12, 13, 14, 15, 16, 18, 19, 21, 22, 23, 24, 27, 31, 32, 33, 35, 36, 38, 40, 69, 70], "hf_token": 3, "agreement": 3, "issu": [3, 5, 14, 70, 71], "attempt": [3, 5, 14], "ue": 3, "befor": [3, 10, 12, 13, 14, 15, 18, 19, 20, 37, 69, 70, 71], "relat": [3, 14, 18, 37, 70, 71], "consol": 3, "output": [3, 5, 7, 10, 12, 13, 14, 15, 18, 36, 37, 40, 70, 71], "point": [3, 5, 10, 12, 13, 14, 15, 16, 33, 37, 69, 70, 73], "As": [3, 14, 15, 69, 70, 71], "23": [3, 70, 71], "24": [3, 14, 69, 70, 71, 72], "current": [3, 5, 10, 12, 13, 14, 15, 18, 19, 24, 36, 70, 71], "co": [3, 69], "mistralai": [3, 38], "mixtral": [3, 5, 6, 38, 42, 72], "8x7b": [3, 38], "v0": [3, 38, 72], "mistral": [3, 15, 18, 35, 38, 42, 72], "7b": [3, 5, 7, 38, 71, 72], "instruct": [3, 38, 71, 72], "mean": [4, 5, 10, 14, 15, 16, 18, 19, 24, 28, 29, 36, 37, 69, 70, 71], "": [4, 5, 7, 10, 11, 12, 13, 14, 15, 16, 18, 35, 36, 37, 38, 40, 68, 69, 71, 73], "both": [4, 5, 10, 14, 15, 18, 36, 37, 39, 66, 70, 71], "low": [4, 11, 14, 15, 18, 69, 71], "hang": [4, 71], "fruit": [4, 71], "bar": [4, 14], "entri": [4, 18, 39, 40, 71], "The": [4, 5, 7, 10, 11, 12, 13, 14, 15, 16, 18, 19, 22, 24, 27, 31, 32, 33, 35, 36, 37, 38, 39, 40, 63, 68, 69, 70, 71, 73], "standard": [4, 6, 14, 15, 70, 71], "answer": [4, 10, 40, 69, 70, 71], "why": [4, 5, 10, 18, 69, 70, 71], "yet": [4, 5, 12, 13, 14, 70, 71, 73], "aren": [4, 38, 71], "t": [4, 5, 10, 11, 12, 13, 14, 15, 16, 18, 35, 38, 69, 70, 71, 73], "enough": [4, 5, 10, 70, 71], "peopl": [4, 5, 71], "guid": [4, 71], "arena": 4, "callum": [4, 71], "mcdougal": [4, 71], "comprehens": [4, 71], "introduct": 4, "mech": [4, 70], "interp": [4, 70], "written": [4, 5, 7, 70], "snippet": 4, "copi": [4, 12, 13, 70], "come": [4, 5, 14, 15, 40, 70, 71], "exercis": [4, 70], "solut": [4, 5, 70, 71], "notabl": [4, 14, 37, 70, 71], "video": [4, 7, 70, 71], "me": [4, 5, 38, 71, 73], "good": [4, 5, 7, 10, 35, 69, 70, 71, 73], "cover": [4, 14, 71], "foundat": [4, 71], "concept": [4, 70, 71], "wild": [4, 10, 70, 71], "techniqu": [4, 7, 14, 40, 70, 71], "direct": [4, 5, 10, 14, 16, 37, 40, 71], "logit": [4, 5, 6, 7, 10, 12, 13, 14, 15, 22, 35, 40, 69, 71], "patch": [4, 5, 7, 8, 9], "paper": [4, 7, 10, 14, 15, 18, 32, 35, 40, 71], "read": [4, 5, 7, 10, 14, 71], "200": [4, 71], "explain": [4, 7, 70, 71], "jargon": 4, "unfamiliar": [4, 70], "term": [4, 10, 14, 70], "go": [4, 5, 7, 40, 71, 73], "across": [4, 5, 10, 12, 13, 14, 16, 38, 40, 68, 70, 71], "youtub": 4, "channel": 4, "content": [4, 35, 70, 71], "walkthrough": [4, 70, 71], "am": 5, "happi": 5, "announc": 5, "now": [5, 7, 14, 15, 70, 71], "releas": 5, "recent": 5, "primari": 5, "motiv": [5, 70], "behind": [5, 18, 70], "jump": [5, 71], "transit": [5, 70], "strictli": [5, 10, 71], "describ": [5, 15, 69, 70], "At": [5, 70], "last": [5, 10, 14, 69, 71], "minut": 5, "did": [5, 10, 69, 70, 71], "remov": [5, 10, 11, 14, 18, 37, 69, 70, 71, 73], "hookedsa": 5, "had": [5, 70, 71], "saelen": 5, "bundl": [5, 71], "major": 5, "hand": [5, 70, 71], "modif": 5, "affect": [5, 14, 15, 40, 70], "bryce": 5, "meyer": 5, "softwar": 5, "engin": [5, 40, 70, 71, 73], "littl": [5, 15, 71, 73], "under": [5, 10, 14, 24], "15": [5, 70, 71], "profession": [5, 7], "experi": [5, 7, 15, 70, 71, 73], "wide": 5, "expertis": 5, "embed": [5, 7, 10, 12, 13, 14, 15, 18, 21, 22, 27, 33, 70, 71], "comput": [5, 10, 11, 14, 20, 21, 22, 23, 25, 26, 28, 29, 30, 32, 34, 36, 37, 39, 40, 69, 70, 71, 73], "coupl": 5, "gotten": [5, 14], "ml": [5, 70, 71, 73], "especi": [5, 14, 70, 71], "ai": [5, 15, 18, 38, 69, 71], "safeti": 5, "nine": 5, "march": 5, "chat": [5, 38, 71, 72], "bit": [5, 15, 70, 71], "he": [5, 71], "ask": 5, "might": [5, 10, 12, 13, 70], "interest": [5, 7, 12, 13, 14, 70, 71], "take": [5, 7, 10, 14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 40, 69, 70, 71, 73], "role": 5, "maintain": 5, "basi": [5, 12, 14], "april": 5, "far": [5, 14, 69, 70, 71], "pretti": [5, 10, 14, 69, 70, 71], "mani": [5, 14, 24, 39, 40, 63, 70, 71], "kind": [5, 10, 70, 71], "address": 5, "everi": [5, 10, 12, 13, 14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 40, 63, 69, 70, 71], "await": 5, "repli": 5, "total": [5, 70, 71], "around": [5, 6, 10, 12, 13, 14, 37, 40, 68, 70, 71], "30": [5, 70, 71, 72], "20": [5, 69, 70, 71, 72], "pr": 5, "were": [5, 10, 14, 15, 35, 69, 70, 71, 73], "limit": [5, 12, 13, 14, 70], "llama": [5, 7, 15, 38, 42, 72], "quantiz": [5, 15], "hookedsaetransform": 5, "brand": 5, "class": [5, 10, 11, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 37, 38, 39, 63, 69], "splice": 5, "autoencod": 5, "two": [5, 6, 11, 12, 32, 36, 38, 40, 69, 70, 71], "goal": [5, 70, 71, 73], "posit": [5, 7, 10, 12, 13, 14, 15, 18, 21, 27, 30, 33, 35, 36, 37, 38, 40, 69, 70, 71], "while": [5, 12, 13, 14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 70, 71], "remain": [5, 10, 14, 37, 71], "power": [5, 71], "who": 5, "push": 5, "second": [5, 14, 35, 70, 71], "base": [5, 7, 10, 11, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 37, 38, 39, 63, 68, 69, 70, 71, 72], "easier": [5, 14, 35, 70, 71, 73], "futur": 5, "llm": 5, "continu": [5, 40, 70, 71], "acceler": [5, 70, 71], "feel": [5, 70, 71, 73], "massiv": [5, 70, 71], "amount": [5, 10, 15, 70], "momentum": [5, 63], "moment": [5, 18, 19, 24], "hope": [5, 71], "carri": 5, "over": [5, 10, 14, 40, 69, 70, 71], "background": [5, 70], "know": [5, 7, 10, 12, 13, 70, 71], "talk": [5, 71], "ensur": [5, 14, 70], "meet": 5, "person": [5, 70], "spoken": 5, "dozen": 5, "commun": 5, "happen": [5, 70, 71], "appoint": 5, "curiou": 5, "hear": 5, "anyon": [5, 35], "tool": [5, 7, 71, 73], "absolut": [5, 6, 12, 13, 14, 15, 18, 27, 36, 69, 70, 71], "beginn": 5, "complet": [5, 10, 18, 69, 70, 71], "expert": [5, 15], "Not": [5, 18, 19, 24, 26], "idea": [5, 14, 18, 40, 70, 71, 73], "evolv": 5, "biggest": [5, 10], "previous": [5, 70], "offici": [5, 38, 71], "instanc": [5, 12, 13, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 36, 37, 70], "compat": [5, 14, 38, 73], "through": [5, 14, 37, 70, 71], "forward": [5, 10, 12, 13, 14, 15, 18, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 37, 39, 71], "start": [5, 10, 13, 14, 15, 18, 40, 69, 70, 71, 73], "todai": [5, 71, 73], "assur": 5, "abl": [5, 70, 71], "upgrad": 5, "worri": 5, "There": [5, 6, 10, 12, 13, 36, 38, 69, 70, 71, 73], "right": [5, 12, 13, 14, 18, 19, 40, 69, 70, 71], "move_model": [5, 10], "activationcach": [5, 8, 9, 12, 13, 14, 36, 40, 69, 70, 71], "function": [5, 10, 12, 13, 14, 15, 18, 20, 21, 22, 23, 24, 25, 26, 28, 29, 32, 34, 36, 37, 38, 40, 63, 65, 66, 68, 69, 71, 73], "cache_al": [5, 37], "hook_point": [5, 8, 9, 14, 71], "keep": [5, 10, 14, 37, 70, 71, 73], "thing": [5, 11, 14, 15, 18, 40, 70, 71, 73], "simpl": [5, 70, 71], "howev": [5, 10, 14, 24, 35, 70, 71], "them": [5, 10, 14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 40, 69, 70, 71], "adapt": [5, 71], "awai": [5, 14, 70], "along": [5, 14, 18, 69, 70, 71], "anyth": [5, 35, 70], "mark": [5, 70, 71], "whenev": [5, 37], "someth": [5, 14, 35, 70], "becom": [5, 70, 71], "promin": 5, "sort": [5, 37, 71], "slip": 5, "scenario": 5, "situat": 5, "handl": [5, 14, 37], "persist": [5, 37], "allow": [5, 14, 36, 40, 69, 70, 71], "reli": 5, "interrupt": 5, "still": [5, 37, 70], "encourag": [5, 18, 69], "everyon": 5, "period": [5, 70, 71], "ey": 5, "don": [5, 10, 12, 13, 14, 15, 16, 35, 69, 70, 71, 73], "imagin": [5, 70], "often": [5, 10, 14, 15, 38, 69, 70, 71], "save": [5, 10, 14, 15, 36, 63, 69, 70, 71], "troubl": [5, 10], "move": [5, 10, 12, 13, 14, 40, 70, 71], "three": [5, 14, 40, 69, 70], "timefram": 5, "plan": 5, "state": [5, 10, 14, 31, 37, 70, 71, 73], "tracker": 5, "categor": 5, "easi": [5, 10, 14, 69, 70, 71, 73], "date": [5, 14], "below": [5, 12, 13, 14, 70], "draft": 5, "our": [5, 33, 70, 71, 73], "priorit": 5, "feedback": [5, 69, 70, 71, 73], "surfac": 5, "other": [5, 10, 12, 14, 18, 19, 24, 36, 37, 38, 40, 70], "improv": [5, 69, 70, 71], "achiev": [5, 10, 71], "diagnos": 5, "variou": [5, 15, 37, 70, 71, 73], "area": 5, "memori": [5, 10, 11, 12, 13, 14, 15, 70, 71], "leak": 5, "occur": [5, 70], "seem": [5, 14, 15, 35, 38, 70, 71], "refer": [5, 10, 14, 19, 37, 70, 71], "properli": [5, 35, 70], "thu": 5, "caus": 5, "garbag": 5, "collect": [5, 71], "correctli": [5, 14], "identifi": [5, 7, 14, 40, 70, 71], "proper": 5, "overal": [5, 15, 70], "deal": [5, 14, 18, 69, 70], "larger": [5, 10, 35, 70, 71], "task": [5, 7, 12, 14, 15, 35, 40, 63, 70], "explor": [5, 69, 71], "abil": [5, 10, 40, 70], "batch": [5, 10, 12, 13, 14, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 31, 32, 33, 34, 35, 37, 39, 40, 63, 66, 67, 69, 70, 71], "process": [5, 14, 15, 38, 70, 71], "alreadi": [5, 10, 14, 69, 70, 71], "share": 5, "well": [5, 7, 14, 16, 37, 40, 69, 70, 71], "togeth": [5, 14, 69, 70, 71], "separ": [5, 12, 13, 14, 15, 37, 67, 69, 70, 71], "volunt": 5, "said": [5, 71], "submiss": 5, "discuss": [5, 70], "think": [5, 14, 35, 69, 70, 71], "few": [5, 10, 12, 13, 70, 71], "week": 5, "confus": [5, 70, 71], "among": 5, "calcul": [5, 10, 11, 12, 13, 14, 15, 18, 24, 36, 67, 69, 70, 71], "match": [5, 10, 14, 36, 66, 70], "huggingfac": [5, 12, 13, 14, 15, 35, 38, 66, 69, 71, 73], "solv": [5, 37, 70, 71, 73], "systemat": 5, "submit": 5, "show": [5, 7, 14, 16, 36, 70, 71, 73], "order": [5, 14, 18, 38, 40, 69, 70], "allevi": 5, "build": [5, 37, 71, 73], "spit": 5, "tabl": [5, 70, 71], "u": [5, 11, 14, 40, 69, 70, 71], "snapshot": 5, "store": [5, 10, 14, 15, 24, 37, 39, 40, 63, 70, 71], "repo": [5, 70], "regener": 5, "cumul": [5, 14, 69], "valu": [5, 7, 10, 11, 12, 13, 14, 15, 18, 24, 30, 31, 33, 36, 38, 39, 40, 69, 70, 71, 73], "creat": [5, 7, 10, 14, 18, 37, 70, 71], "robust": [5, 10, 70], "big": [5, 36, 38, 69, 70, 71], "famili": [5, 15, 71], "hard": [5, 14, 70, 71], "even": [5, 7, 11, 14, 15, 18, 35, 38, 70, 71, 73], "smallest": [5, 71], "thought": [5, 10, 70, 71], "thrown": 5, "topic": [5, 7], "best": [5, 7, 14, 70, 71], "guess": [5, 70], "reason": [5, 18, 19, 24, 70, 71], "untrain": 5, "eg": [5, 10, 14, 35, 40, 69, 70, 71], "randomli": [5, 14, 15, 71], "weight": [5, 7, 12, 13, 14, 15, 18, 24, 26, 63, 66, 69, 70, 71, 73], "verifi": [5, 7, 70, 71], "load": [5, 10, 12, 13, 14, 15, 35, 38, 69, 70, 73], "result": [5, 10, 12, 13, 14, 15, 21, 30, 33, 36, 38, 40, 69, 70, 71, 73], "accur": 5, "sens": [5, 11, 37, 70, 71], "consist": [5, 14, 70, 71], "sampl": [5, 14, 35, 69], "size": [5, 10, 14, 15, 18, 35, 63, 69, 70, 71], "against": [5, 70], "bite": 5, "success": [5, 70], "turn": [5, 10, 14, 69, 70, 71], "effici": [5, 11, 18, 69, 71], "proof": [5, 70], "put": [5, 70, 71], "strong": 5, "opinion": 5, "most": [5, 10, 14, 37, 69, 70, 71, 73], "roundtabl": 5, "wrapper": [5, 10, 12, 13, 14, 40, 68, 71], "plugin": 5, "addit": [5, 7, 12, 13, 14, 70, 71], "outsid": 5, "publish": 5, "themselv": [5, 14], "final": [5, 6, 10, 12, 13, 14, 15, 18, 69, 70, 71], "overhaul": 5, "composit": [5, 14, 70, 71], "util": [5, 8, 9, 10, 11, 14, 36, 37, 38, 63, 70, 71], "isol": [5, 70], "rapidli": 5, "itself": [5, 36, 69, 70], "none": [5, 10, 12, 13, 14, 15, 16, 18, 19, 20, 21, 24, 25, 27, 28, 30, 31, 33, 35, 36, 37, 38, 40, 63, 68, 69, 70, 71], "pain": [5, 71], "rel": [5, 15, 30, 70], "grow": 5, "exponenti": 5, "whole": [5, 70, 71], "explod": 5, "section": [5, 15, 70, 71], "relev": [5, 14, 15, 18, 40, 69, 70, 71], "skip": [5, 14, 70, 71], "setup": [5, 14, 37, 40], "act": [5, 37, 40, 69, 70, 71], "vast": 5, "due": [5, 6, 14, 71], "potenti": 5, "mismatch": [5, 36], "between": [5, 10, 14, 15, 18, 36, 40, 69, 70, 71, 73], "meant": 5, "repres": [5, 11, 12, 15, 32, 36, 40, 69, 70, 71], "updat": [5, 7, 14, 39, 40, 68, 70, 71], "readi": 5, "sent": 5, "justifi": 5, "bug": [5, 7, 10, 15, 71], "fix": [5, 37, 70, 71], "exist": [5, 14, 36, 38, 70, 71], "split": [5, 14, 18, 38, 69, 70, 71], "group": [5, 14, 15, 18, 24], "call": [5, 10, 12, 13, 14, 15, 20, 21, 22, 23, 24, 25, 26, 28, 29, 32, 34, 37, 38, 69, 70, 71], "again": [5, 70], "piec": [5, 70], "tradition": 5, "everyth": [5, 37, 40, 69, 71], "That": [5, 70], "mock": 5, "spi": 5, "control": [5, 15, 40, 70, 71], "input": [5, 10, 12, 13, 14, 15, 24, 27, 32, 37, 38, 39, 40, 66, 67, 69, 70, 71], "side": [5, 11, 14], "effect": [5, 10, 14, 15, 40, 70, 71], "certain": [5, 15, 40], "logic": 5, "entir": [5, 10, 14, 40, 70], "rule": 5, "incredibli": [5, 71], "cannot": [5, 14, 69, 71], "origin": [5, 15, 16, 18, 70, 71], "pass": [5, 10, 14, 15, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 32, 33, 34, 36, 37, 38, 39, 69, 71], "bunch": [5, 10, 12, 13, 14, 37, 70, 71], "Being": [5, 70], "live": [5, 12, 13, 14, 70, 71], "ci": 5, "report": 5, "great": [5, 7, 71, 73], "get": [5, 7, 10, 12, 13, 14, 15, 16, 18, 35, 36, 37, 40, 68, 70, 71, 73], "review": 5, "meaning": [5, 15, 40], "than": [5, 6, 10, 14, 15, 18, 20, 35, 36, 37, 40, 69, 70, 71], "ever": [5, 69], "substanti": 5, "let": [5, 15, 69, 70, 71, 73], "shift": [5, 40, 71], "favor": 5, "individu": [5, 7, 10, 14, 18, 70], "3000": 5, "18": [5, 69, 70, 71, 72], "distinct": 5, "either": [5, 14, 36, 37, 38, 40, 70, 71], "interdepend": 5, "least": [5, 40, 71], "depend": [5, 10, 14, 71], "its": [5, 10, 14, 18, 37, 70, 71, 73], "own": [5, 7, 70, 71], "realli": [5, 35, 36, 69, 70, 71], "anywher": [5, 40], "els": [5, 14, 15, 18, 38, 69, 70, 71], "mlp": [5, 10, 12, 13, 14, 15, 20, 33, 40, 69, 70, 71], "exactli": [5, 14, 38, 66, 70, 71], "thank": [5, 71], "excit": 5, "standpoint": 5, "perspect": [5, 71], "comparison": [5, 36], "worth": [5, 7, 10, 70, 71], "enabl": [5, 10, 14, 70, 71, 73], "huge": 5, "impact": [5, 10], "bring": 5, "realiti": [5, 70], "semver": 5, "older": [5, 71], "log": [5, 14, 63, 69, 70, 71], "data": [5, 6, 7, 14, 35, 69, 70, 71], "expos": [5, 14, 73], "properti": [5, 11, 12, 13, 14, 18, 24, 69, 70, 71], "minor": [5, 70], "bump": [5, 71], "whatsoev": 5, "With": [5, 18, 71], "fact": [5, 14, 70, 71, 73], "discov": 5, "earlier": [5, 70, 71], "extent": 5, "probabl": [5, 7, 14, 35, 36, 40, 69, 70, 71], "regardless": [5, 69], "stand": 5, "reliabl": 5, "17": [5, 70, 71], "possibli": 5, "easiest": [5, 70], "fresh": 5, "consum": [5, 10, 11], "top": [6, 14, 69, 70, 71], "k": [6, 10, 11, 12, 13, 14, 16, 18, 24, 40, 69, 70, 71], "gate": [6, 14], "hidden": [6, 15, 31, 71], "amplifi": 6, "greatli": [6, 71, 73], "select": [6, 10, 69, 70, 71], "lead": [6, 7, 12, 13, 37, 69, 71], "higher": [6, 14, 70], "normal": [6, 10, 14, 15, 26, 69, 70, 71, 73], "varianc": [6, 70], "test": [6, 7, 14, 35, 36, 69, 70, 71], "half": [6, 11, 12, 13, 14, 18, 35, 71], "precis": [6, 36, 40, 70, 71], "deviat": [6, 15, 71], "compar": [6, 35, 69, 71, 73], "2e": 6, "mitig": 6, "disabl": [6, 14, 36, 37, 70], "preprocess": [6, 12, 13, 70], "option": [6, 10, 12, 13, 14, 15, 16, 18, 19, 20, 21, 24, 25, 27, 28, 30, 31, 33, 35, 36, 37, 38, 39, 40, 63, 68, 69], "from_pretrained_no_process": [6, 14], "increas": [6, 40, 70, 71], "colab": [7, 70, 71, 73], "blob": [7, 66], "ipynb": 7, "causal": [7, 12, 15, 40, 70, 71], "intervent": [7, 40, 70, 71], "matter": [7, 14, 40, 70, 71], "produc": [7, 14, 40, 70], "incomplet": 7, "gradient": [7, 10, 37, 63, 71], "approxim": [7, 70, 71], "bad": [7, 14], "residu": [7, 10, 12, 13, 14, 15, 18, 31, 33, 34, 40, 71], "stream": [7, 10, 12, 13, 14, 15, 31, 33, 40, 69, 71], "after": [7, 10, 14, 15, 20, 26, 37, 63, 70, 71, 73], "demonstr": [7, 16, 70, 71], "focus": [7, 70, 71], "less": [7, 14, 18, 70, 71], "rigor": [7, 70, 71], "grasp": 7, "steal": 7, "liber": [7, 37], "phenomenon": 7, "memoris": 7, "minimis": 7, "loss": [7, 10, 14, 35, 37, 40, 63, 69, 70, 71], "longer": 7, "generalis": [7, 70, 71], "sharp": [7, 71], "decreas": [7, 18, 69, 70], "modular": [7, 69], "grok": 7, "light": 7, "explan": [7, 40, 70], "ll": [7, 14, 36, 70, 71], "pair": [7, 11, 14, 18, 36, 69, 70, 71], "seri": [7, 10, 71], "detector": [7, 36], "detect": [7, 36, 70, 71], "sever": [7, 10, 14, 69, 70, 71], "custom": [7, 14, 15, 21, 35, 37, 69, 70, 71], "algorithm": [7, 11, 15, 71, 73], "interact": [7, 65, 70, 71], "neuroscop": [7, 71], "hacki": [7, 69], "web": [7, 71], "visualis": [7, 70], "front": 7, "visual": [7, 12, 13, 71], "dynam": [7, 15, 71], "convert": [7, 12, 13, 14, 38, 69, 70, 71], "meta": [7, 14, 38, 69, 70, 71, 72], "until": [7, 10, 14, 37, 70, 71], "multi": [7, 10, 69, 71], "gpu": [7, 10, 11, 12, 13, 14, 70, 71], "access": [7, 10, 15, 37, 69, 70], "No": [7, 71], "previou": [7, 10, 14, 31, 33, 36, 70, 71], "port": 7, "excel": [7, 10, 70, 71, 73], "sequenc": [7, 12, 13, 14, 15, 18, 32, 35, 36, 37, 40, 69, 70, 71], "investig": [7, 10, 14, 36, 70, 71], "svd": [7, 11, 14, 16, 71], "conjectur": 7, "post": [7, 10, 15, 16, 70, 71], "singular": [7, 11, 14, 16, 71], "decomposit": [7, 10, 11, 14, 70, 71], "matric": [7, 11, 12, 13, 14, 16, 18, 19, 24, 66, 70, 71, 73], "surprisingli": 7, "reproduc": [7, 15, 36], "further": [7, 10, 14, 69, 70, 71], "tracr": 7, "cool": 7, "deepmind": 7, "compil": 7, "program": [7, 71, 73], "rasp": 7, "jax": 7, "form": [7, 10, 11, 14, 40, 70, 71], "pytorch": [7, 14, 15, 35, 37, 71], "brows": 8, "first": [8, 10, 14, 15, 35, 38, 40, 69, 70, 71], "submodul": 8, "factoredmatrix": [8, 9, 12, 13, 18, 69, 71], "hookedencod": [8, 9, 68], "hookedencoderdecod": [8, 9, 68], "hookedtransformerconfig": [8, 9, 14, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 38, 39, 44, 45, 51, 52, 58, 59, 60, 61, 62, 68], "svdinterpret": [8, 9], "eval": [8, 9, 71], "head_detector": [8, 9], "past_key_value_cach": [8, 9], "subpackag": 8, "pretrain": [8, 9, 10, 12, 13, 14, 15, 35, 38, 69, 70, 71], "core": [10, 14, 70, 71, 73], "varieti": [10, 71], "helper": [10, 14, 18, 35, 37, 40, 69, 71], "skim": 10, "method": [10, 12, 13, 14, 15, 37, 38, 39, 69, 70, 71], "back": [10, 15, 18, 19, 24, 71], "cache_dict": 10, "dict": [10, 12, 13, 14, 15, 18, 19, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 32, 33, 34, 35, 36, 37, 38, 69], "str": [10, 12, 13, 14, 15, 18, 19, 24, 30, 35, 36, 37, 38, 39, 40, 63, 68, 69, 70, 71], "tensor": [10, 11, 12, 13, 14, 16, 18, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 36, 37, 39, 40, 66, 67, 69, 70, 71, 73], "has_batch_dim": 10, "bool": [10, 12, 13, 14, 15, 30, 31, 35, 36, 37, 38, 39, 40, 63, 69], "run_with_cach": [10, 12, 13, 14, 37, 70, 71], "particular": [10, 16, 36, 70, 71], "behaviour": [10, 14, 70, 71], "modal": 10, "step": [10, 14, 15, 38, 63, 69, 70, 71], "respons": [10, 15, 70], "prompt": [10, 14, 18, 35, 40, 69, 70, 71], "chicken": 10, "road": [10, 69], "specif": [10, 12, 13, 14, 18, 36, 38, 40, 70, 71], "sublay": 10, "commonli": 10, "fall": 10, "categori": [10, 70], "dla": 10, "_logit": 10, "residual_stream": 10, "label": [10, 14, 15, 18, 19, 24, 38, 70, 71], "decompose_resid": [10, 70], "return_label": [10, 70], "0": [10, 12, 13, 14, 15, 16, 18, 27, 32, 36, 38, 63, 69, 70, 71, 72], "emb": [10, 17, 21, 27, 32, 69, 71], "pos_emb": [10, 15, 17, 71], "0_attn_out": 10, "proceed": 10, "space": [10, 12, 14, 69, 70, 71], "logit_attr": 10, "shape": [10, 12, 14, 18, 19, 24, 31, 32, 33, 40, 69, 70, 71], "torch": [10, 12, 13, 14, 15, 18, 24, 27, 31, 33, 37, 38, 40, 66, 68, 69, 70, 71], "10": [10, 14, 16, 69, 70, 71, 72], "7": [10, 35, 70, 71, 72], "most_important_component_idx": 10, "argmax": [10, 70], "3_attn_out": 10, "dig": [10, 70, 71, 73], "granular": 10, "get_full_resid_decomposit": 10, "stack": [10, 12, 13, 14, 40, 69, 70, 71], "equal": [10, 15, 71], "struggl": 10, "construct": [10, 12, 13], "joke": 10, "trivial": 10, "accumulated_resid": [10, 70], "footgun": [10, 37], "sourc": [10, 14, 15, 18, 35, 40, 73], "track": [10, 70], "index": [10, 12, 13, 14, 15, 16, 18, 19, 24, 38, 40, 68, 69, 70, 71], "dimens": [10, 14, 15, 18, 24, 25, 28, 37, 40, 66, 67, 69, 70, 71], "vector": [10, 11, 14, 16, 18, 40, 70, 71], "q": [10, 12, 13, 14, 18, 24, 40], "z": [10, 14, 24, 36, 40, 70, 71], "po": [10, 12, 13, 14, 18, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 40, 67, 69, 70, 71], "head_index": [10, 14, 16, 18, 19, 24, 25, 26, 30, 31, 40, 67, 70, 71], "d_head": [10, 12, 13, 14, 15, 18, 19, 24, 38, 39, 67, 69, 70, 71, 72], "pattern": [10, 12, 13, 14, 18, 19, 24, 36, 40, 71], "softmax": [10, 14, 15, 18, 19, 24, 26, 69, 71], "attn_scor": [10, 15, 18, 19], "pre": [10, 14, 15, 18, 19, 24, 26, 29, 36, 69], "query_po": [10, 18, 19, 24, 70], "key_po": [10, 18, 19, 24, 70], "d_model": [10, 12, 13, 14, 15, 18, 19, 20, 22, 23, 24, 25, 26, 27, 28, 31, 33, 34, 38, 67, 69, 70, 71, 72], "mid": [10, 70], "solu_ln": [10, 15], "layernorm": [10, 12, 13, 14, 15, 20, 25, 26, 28, 69, 70], "d_mlp": [10, 12, 13, 14, 15, 26, 38, 69, 71, 72], "resid_pr": [10, 15, 20, 31, 33, 40, 70, 71], "resid_mid": [10, 40], "resid_post": [10, 15, 70], "attn_out": [10, 14, 15, 40, 70], "mlp_out": [10, 14, 15, 40, 70], "ln": [10, 14, 15, 70, 71], "lnpre": [10, 15], "scale": [10, 14, 15, 18, 19, 24, 69, 70, 71], "sometim": [10, 35, 70], "miss": [10, 70], "becaus": [10, 11, 12, 13, 14, 15, 18, 35, 69, 70, 71, 73], "appli": [10, 14, 15, 18, 20, 33, 37, 40, 69, 70, 71], "remove_batch_dim": [10, 37, 69, 71], "batch_siz": [10, 12, 32, 35, 37, 39, 63, 70, 71], "annot": [10, 71], "layers_cov": 10, "queri": [10, 12, 13, 14, 15, 18, 19, 24, 30, 36, 40, 71], "batch_and_pos_dim": 10, "ve": [10, 14, 18, 35, 37, 70, 73], "slice": [10, 37, 69, 70], "dictionari": [10, 12, 13, 14, 15, 36, 37, 38, 69, 71], "whether": [10, 12, 14, 15, 18, 32, 35, 37, 38, 40, 63, 69, 70, 71], "int": [10, 11, 12, 13, 14, 15, 16, 18, 19, 21, 23, 24, 25, 27, 28, 30, 31, 32, 33, 35, 36, 37, 38, 39, 40, 63, 68, 69, 70, 71], "incl_mid": [10, 70], "fals": [10, 12, 13, 14, 15, 30, 35, 36, 37, 38, 39, 40, 63, 69, 70, 71], "apply_ln": [10, 70], "pos_slic": [10, 37, 70], "union": [10, 12, 13, 14, 15, 16, 18, 19, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 32, 33, 34, 36, 37, 38, 39, 40, 68, 69, 70], "tupl": [10, 11, 12, 13, 14, 18, 24, 36, 37, 40, 69, 71], "ndarrai": [10, 14, 37, 69], "mlp_input": [10, 14], "float": [10, 11, 12, 13, 14, 15, 18, 20, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 33, 34, 36, 38, 39, 40, 63, 66, 67, 69, 70, 71], "accumul": [10, 14, 70], "sub": [10, 71], "www": 10, "lesswrong": 10, "ackrb8wdpdan6v6ru": 10, "believ": [10, 35, 70], "vocabulari": [10, 15, 70, 71], "rememb": 10, "norm": [10, 11, 14, 15, 21, 25, 26, 28, 29, 38, 63, 70, 71], "decod": [10, 13, 15, 31], "therefor": [10, 14, 69], "multipli": [10, 14, 18, 19, 36, 66, 70, 71], "unembed": [10, 12, 13, 14, 70, 71], "matrix": [10, 11, 12, 13, 14, 15, 16, 18, 24, 36, 69, 70], "w_u": [10, 12, 13, 14, 70, 71], "broken": [10, 37, 69, 70, 71], "down": [10, 14, 18, 19, 24, 70, 71], "einop": [10, 70, 71], "einsum": [10, 70, 71], "panda": [10, 40], "pd": [10, 40], "devic": [10, 12, 13, 14, 15, 16, 18, 30, 35, 37, 38, 39, 63, 64, 70, 71], "answer_token": [10, 70], "to_single_token": [10, 14, 70, 71], "2975": 10, "accum_resid": 10, "last_token_accum": 10, "9": [10, 69, 70, 71, 72], "64": [10, 38, 69, 71, 72], "50257": [10, 38, 71, 72], "layers_unembed": 10, "d_vocab": [10, 12, 13, 14, 15, 16, 38, 40, 69, 71, 72], "rank": [10, 11, 14, 18, 69, 70, 71], "correct": [10, 40, 68, 69, 70, 71], "sorted_indic": 10, "argsort": 10, "dim": [10, 14, 69, 70, 71], "descend": [10, 71], "rank_answ": 10, "nonzero": 10, "as_tupl": 10, "0_pre": 10, "4442": 10, "1_pre": [10, 70], "382": 10, "2_pre": 10, "982": 10, "3_pre": 10, "1160": 10, "4_pre": 10, "408": 10, "5_pre": 10, "145": 10, "6_pre": 10, "78": 10, "7_pre": 10, "387": 10, "final_post": 10, "6": [10, 14, 38, 69, 70, 71, 72], "dtype": [10, 12, 13, 14, 15, 18, 38, 68, 70, 71], "int64": [10, 69], "exclud": [10, 36], "n_layer": [10, 12, 13, 14, 15, 38, 40, 68, 70, 71, 72], "immedi": [10, 19, 69, 70, 71], "indic": [10, 12, 13, 32, 37, 40, 69, 70, 71], "taken": [10, 14, 71], "l": [10, 12, 13, 14, 70, 71], "noth": [10, 12, 13, 14, 37, 69, 70, 71], "essenti": [10, 14, 70, 71, 73], "rather": [10, 14, 15, 20, 40, 69, 70, 71], "graph": [10, 70, 71], "apply_ln_to_stack": [10, 14, 69, 70], "residual_stack": [10, 70], "num_compon": 10, "batch_slic": 10, "batch_and_pos_dims_out": 10, "treat": [10, 14, 15, 70, 71], "factor": [10, 11, 14, 15, 70], "simul": [10, 14, 70, 71], "global": [10, 14, 18, 19, 24, 30, 37, 38, 69, 70, 71], "element": [10, 14, 18, 36, 40, 69, 71], "rmsnorm": [10, 15, 28], "unchang": [10, 12, 13, 14, 69, 70, 71], "whose": [10, 12, 13, 14, 35, 69, 70], "trail": [10, 11, 69], "assum": [10, 12, 13, 14, 15, 25, 28, 32, 37, 40, 63, 69, 70], "hook_scal": [10, 69, 70, 71], "unemb": [10, 14, 15, 17, 70, 71], "map": [10, 12, 13, 14, 15, 18, 36, 37, 70, 71], "ie": [10, 12, 13, 14, 15, 18, 38, 40, 69, 70, 71], "ln2": [10, 33, 69, 71], "ln1": [10, 15, 33, 69, 71], "ln_final": [10, 14, 70, 71], "apply_slice_to_batch_dim": 10, "compute_head_result": 10, "sum": [10, 11, 14, 15, 21, 36, 69, 70, 71], "plu": 10, "b_o": [10, 12, 13, 14, 71], "intend": [10, 15, 69], "use_attn_result": [10, 14, 15], "forget": 10, "liter": [10, 12, 13, 14, 16, 36, 37, 40], "incl_emb": 10, "decompos": 10, "incl": 10, "expand_neuron": 10, "bias": [10, 12, 13, 14, 15, 63, 70], "expand": [10, 14, 24], "get_neuron_result": 10, "neuron_slic": 10, "num_neuron": 10, "subset": [10, 15, 35, 70, 71], "specifi": [10, 12, 13, 14, 15, 24, 35, 36, 37, 68, 69, 71], "expens": [10, 11, 71], "cheap": 10, "hook_emb": [10, 69, 71], "hook_pos_emb": [10, 71], "block": [10, 14, 15, 18, 19, 20, 24, 31, 33, 40, 69, 70, 71], "hook_resid_pr": [10, 71], "incorrect_token": [10, 70], "typic": [10, 12, 14, 32, 36, 70, 71], "revers": [10, 11, 40, 69, 70, 71, 73], "dot": [10, 14, 18, 19, 69], "product": [10, 11, 12, 13, 18, 19, 71], "incorrect": [10, 14, 40, 70, 71], "arxiv": [10, 14, 15, 18, 24, 32, 35], "org": [10, 14, 15, 18, 24, 32, 35, 71, 73], "ab": [10, 11, 14, 24, 36, 70, 71], "2211": [10, 35], "00593": [10, 35], "john": [10, 70, 71], "mari": [10, 70, 71], "went": [10, 70, 71], "shop": [10, 70, 71], "gave": [10, 35, 70, 71], "bag": [10, 70], "choos": [10, 70, 71], "final_ln": 10, "residual_stack_item": 10, "dure": [10, 15, 24, 37, 39, 71, 73], "stack_activ": 10, "activation_nam": [10, 40, 70], "sublayer_typ": 10, "flexibl": 10, "given": [10, 11, 12, 13, 14, 16, 36, 37, 38, 40, 68, 69, 70, 71], "get_act_nam": [10, 69, 70, 71], "infer": [10, 14, 26, 40, 70, 71], "incl_remaind": 10, "stack_head_result": [10, 70], "axi": [10, 18, 40, 69, 70, 71], "n_head": [10, 12, 13, 14, 15, 18, 24, 38, 39, 40, 69, 70, 71, 72], "notat": [10, 70, 73], "l0h0": 10, "stack_neuron_result": 10, "l0n0": 10, "super": [10, 14, 71], "short": [10, 69, 70, 71, 73], "mostli": [10, 70, 71], "finish": [10, 14, 69, 70, 71], "oper": [10, 70, 71], "slower": 10, "unless": [10, 14, 15, 35, 71], "deprec": 10, "toggle_autodiff": 10, "toggl": [10, 14], "autodiff": [10, 71], "set_grad_en": [10, 70, 71], "danger": 10, "off": [10, 14, 35, 69, 70, 71], "realis": [10, 30, 70], "downstream": 10, "delet": [10, 69, 70], "stick": [10, 70], "mess": [10, 14, 69, 71], "inference_mod": 10, "decor": 10, "similar": [10, 12, 13, 14, 20, 24, 36, 70, 71], "requires_grad": 10, "eigenvalu": 11, "ldim": [11, 71], "mdim": [11, 71], "rdim": [11, 71], "leading_dim": [11, 69], "ba": 11, "vh": [11, 14], "collapse_l": 11, "collaps": [11, 70, 71], "left": [11, 14, 18, 69, 70, 71, 73], "orthogon": [11, 14], "self": [11, 12, 13, 14, 18, 69, 71], "collapse_r": 11, "analog": [11, 70, 71], "apart": [11, 69, 70, 71], "zero": [11, 14, 18, 36, 69, 70, 71], "bav": 11, "kv": 11, "abav": 11, "kav": 11, "av": 11, "eigenvector": [11, 71], "get_corn": [11, 69, 70], "make_even": 11, "sqrt": [11, 14, 15, 69], "diag": 11, "equival": [11, 14, 18, 70, 71], "factoris": [11, 14, 18, 71], "row": [11, 14, 40], "col": 11, "ndim": 11, "frobeniu": [11, 71], "squar": [11, 28, 29, 69, 71], "m": [11, 18, 66, 69, 70, 71], "st": 11, "transpos": [11, 69], "obviou": [11, 14, 70], "unsqueez": [11, 69], "hook": [12, 13, 14, 15, 19, 20, 21, 22, 23, 25, 26, 27, 28, 29, 32, 33, 34, 37, 70], "encod": [12, 13, 18, 31, 69, 71], "contain": [12, 13, 14, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 32, 33, 34, 35, 36, 37, 38, 39, 69, 70, 71], "bert": [12, 20, 21, 22, 32, 38, 42, 71, 72], "move_to_devic": [12, 13, 14], "kwarg": [12, 13, 14, 38, 69, 70, 71], "hookedrootmodul": [12, 13, 14, 37, 71], "hookpoint": [12, 13, 14, 37, 70, 71], "inherit": [12, 13, 71], "mvp": 12, "mask": [12, 13, 14, 18, 22, 27, 31, 33, 69, 70], "mlm": [12, 22], "next": [12, 14, 69, 70, 71], "sentenc": [12, 13, 14, 22, 32, 36, 70, 71], "nsp": 12, "dropout": [12, 13], "inconsist": [12, 13, 16], "fine": [12, 13, 71], "fold": [12, 13, 14, 18, 26, 38, 70], "ov": [12, 13, 14, 16, 18, 70, 71], "o": [12, 13, 66, 71], "qk": [12, 13, 14, 18, 70], "w_e": [12, 13, 14, 71], "conveni": [12, 13, 14, 15, 37, 69, 71], "w_e_po": [12, 14], "n_ctx": [12, 14, 15, 18, 38, 71, 72], "concaten": [12, 14, 69, 70, 71], "w_po": [12, 13, 14, 71], "overcomplet": [12, 14], "w_k": [12, 13, 14, 15, 18, 24, 71], "w_o": [12, 13, 14, 18, 19, 70, 71], "w_q": [12, 13, 14, 18, 24, 71], "w_v": [12, 13, 14, 18, 24, 71], "w_in": [12, 13, 14, 16, 71], "w_out": [12, 13, 14, 16, 71], "all_head_label": [12, 13, 14], "format": [12, 13, 14, 37, 70, 71], "h": [12, 13, 14, 70, 71], "b_k": [12, 13, 14, 18, 24, 71], "b_q": [12, 13, 14, 71], "b_u": [12, 13, 14, 70, 71], "bia": [12, 13, 14, 15, 18, 28, 29, 30, 66, 70, 71], "b_v": [12, 13, 14, 18, 24, 71], "b_in": [12, 13, 14, 71], "b_out": [12, 13, 14, 71], "buffer": [12, 13, 14], "modifi": [12, 13, 14], "cuda": [12, 13, 14, 15, 35, 38], "associ": [12, 13, 14, 37], "optim": [12, 13, 14, 63, 70], "return_typ": [12, 13, 14, 37, 70, 71], "token_type_id": [12, 21, 32], "one_zero_attention_mask": [12, 13], "binari": [12, 13, 32], "id": [12, 14, 15, 32], "belong": [12, 32], "cl": [12, 32, 71], "sep": [12, 32], "sequence_length": [12, 32, 36, 69], "attend": [12, 13, 15, 18, 19, 24, 69, 70, 71], "ignor": [12, 13, 14, 15, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 69, 71], "primarili": [12, 13], "pad": [12, 13, 14, 18, 27, 31, 33, 69, 70, 71], "shorter": [12, 13, 14, 71], "classmethod": [12, 13, 14, 15, 39, 69], "model_nam": [12, 13, 14, 15, 38, 71], "checkpoint_index": [12, 13, 14, 15, 38, 71], "checkpoint_valu": [12, 13, 14, 15, 38, 71], "hf_model": [12, 13, 14], "float32": [12, 13, 14, 15, 18, 38, 70], "from_pretrained_kwarg": [12, 13, 14], "bertformaskedlm": [12, 13], "unlik": [12, 13, 14, 40, 71], "mp": [12, 13, 14], "model_arg": [12, 13, 14, 37], "return_cache_object": [12, 13, 14], "otherwis": [12, 13, 14, 35, 36, 69], "device_or_dtyp": [12, 13, 14, 68], "print_detail": [12, 13, 14, 68, 69], "cast": [12, 13, 14], "non_block": [12, 13, 14], "memory_format": [12, 13, 14], "channels_last": [12, 13, 14], "Its": [12, 13, 14], "complex": [12, 13, 14, 15, 70, 71], "integr": [12, 13, 14, 38], "tri": [12, 13, 14, 70, 71, 73], "asynchron": [12, 13, 14], "respect": [12, 13, 14, 37, 69, 71], "host": [12, 13, 14, 38], "pin": [12, 13, 14], "desir": [12, 13, 14], "4d": [12, 13, 14], "keyword": [12, 13, 14, 37, 71], "argument": [12, 13, 14, 15, 37, 38, 69, 71], "xdoctest": [12, 13, 14], "ignore_w": [12, 13, 14], "non": [12, 13, 14, 15, 18, 35, 69, 70, 71], "determinist": [12, 13, 14, 69, 70], "nn": [12, 13, 14, 37, 71], "1913": [12, 13, 14], "3420": [12, 13, 14], "5113": [12, 13, 14], "2325": [12, 13, 14], "doubl": [12, 13, 14], "in_featur": [12, 13, 14], "out_featur": [12, 13, 14], "float64": [12, 13, 14], "env": [12, 13, 14], "torch_doctest_cuda1": [12, 13, 14], "gpu1": [12, 13, 14], "1914": [12, 13, 14], "5112": [12, 13, 14], "2324": [12, 13, 14], "float16": [12, 13, 14], "cdoubl": [12, 13, 14], "3741": [12, 13, 14], "j": [12, 13, 14, 15, 18, 38, 70, 71, 72], "2382": [12, 13, 14], "5593": [12, 13, 14], "4443": [12, 13, 14], "complex128": [12, 13, 14], "6122": [12, 13, 14], "1150": [12, 13, 14], "encoderdecod": [13, 15], "t5": [13, 15, 30, 31, 38, 42, 72], "decoder_input": 13, "decoder_po": 13, "usual": [13, 31], "fairli": [14, 70, 71], "extract": [14, 71], "harder": [14, 40, 70], "aim": [14, 70, 73], "simplifi": [14, 70, 71], "attach": [14, 71], "within": [14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 36, 37, 40, 69, 70, 71], "inspect": [14, 70], "alter": 14, "facilit": 14, "deeper": 14, "pretrainedtokenizerbas": 14, "default_padding_sid": 14, "50": [14, 63, 71], "initialis": [14, 15], "although": [14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "instanti": [14, 15, 71], "__init__": [14, 18, 19, 24, 25, 26, 28, 29, 37, 69, 71], "test_prompt": [14, 69, 70, 71], "w_gate": 14, "tokenizer_nam": [14, 15], "explicitli": [14, 15, 18, 35, 38, 40, 71], "n_devic": [14, 15, 38, 68], "greater": [14, 36], "multipl": [14, 36, 68, 69, 71], "accumulated_bia": 14, "include_mlp_bias": 14, "all_composition_scor": [14, 69], "score": [14, 15, 18, 19, 24, 36, 40, 70], "l1": 14, "h1": 14, "l2": 14, "h2": [14, 70], "upper": [14, 18], "triangular": [14, 36, 69, 71], "third": [14, 71], "pub": [14, 69], "2021": 14, "framework": [14, 18, 70, 71], "html": [14, 69, 70], "20abov": 14, "20diagram": 14, "20show": 14, "20q": 14, "2d": [14, 66, 69], "2c": 14, "20k": [14, 69], "20and": 14, "20v": 14, "2dcomposit": 14, "metric": [14, 36, 40, 70, 71], "center_unemb": [14, 70], "state_dict": 14, "center": [14, 15, 26, 28, 29, 70, 71], "subtract": [14, 36, 70], "translat": [14, 70, 71], "invari": 14, "prob": [14, 69, 70, 71], "slightli": [14, 69, 70], "misl": 14, "center_writing_weight": [14, 70, 71], "fold_layer_norm": [14, 38], "check_hooks_to_add": [14, 37], "hook_point_nam": [14, 37], "dir": [14, 37], "fwd": [14, 37], "is_perman": [14, 37], "prepend": [14, 15, 35, 37, 38, 69, 71], "overrid": [14, 15, 37, 38, 69], "fold_bias": 14, "center_weight": 14, "rm": [14, 15, 28, 29], "neighbour": 14, "further_com": [14, 15], "md": [14, 15], "fold_value_bias": 14, "alwai": [14, 15, 40, 70, 71], "constant": [14, 15, 18, 70, 71], "doesn": [14, 35, 69, 70, 71], "formal": 14, "b_o_new": 14, "b_o_origin": 14, "sum_head": 14, "b_v_head": 14, "w_o_head": 14, "loss_per_token": 14, "prepend_bo": [14, 15, 35, 38, 69, 70], "use_default_valu": 14, "padding_sid": [14, 69, 70], "start_at_lay": 14, "shortformer_pos_emb": [14, 18, 33], "attention_mask": [14, 18, 27, 31, 33, 39, 69], "stop_at_lay": [14, 71], "past_kv_cach": [14, 27], "hookedtransformerkeyvaluecach": [14, 31, 33, 39], "flag": [14, 15, 35, 37, 40, 69, 70, 71], "entropi": [14, 69, 70, 71], "per": [14, 40, 70, 71], "averag": [14, 35, 70, 71], "scalar": [14, 18, 37, 71], "default_prepend_bo": [14, 15, 35, 38, 69, 71], "bo": [14, 15, 36, 38, 69, 70, 71], "impli": 14, "usag": [14, 70], "accordingli": [14, 15, 18, 38, 70, 71], "lose": [14, 15, 38], "empir": [14, 15, 38, 40, 71], "inclus": 14, "neg": [14, 69, 70, 71], "shortform": [14, 15, 18, 33, 38], "positional_embedding_typ": [14, 15, 18], "stop": [14, 71], "exclus": [14, 69], "etc": [14, 15, 40, 70, 71, 73], "frozen": [14, 39], "pai": [14, 18, 70], "okai": 14, "twice": [14, 35, 70, 71], "accident": [14, 37], "fold_ln": [14, 38, 70, 71], "refactor_factored_attn_matric": [14, 70], "automodelforcausallm": 14, "first_n_lay": [14, 38], "autoregress": [14, 63], "neo": [14, 18, 19, 24, 38, 42, 71, 72], "gptj": [14, 38, 42], "opt": [14, 38, 42, 71, 72], "solu": [14, 15, 38, 69, 71, 72], "checkpoint": [14, 15, 38, 63], "neelnanda": [14, 38], "stanford": [14, 15, 18, 19, 24, 38, 71, 72], "crfm": [14, 38, 71], "load_and_process_state_dict": 14, "alia": [14, 37, 38, 69, 71], "subsequ": [14, 38, 70, 71], "regular": [14, 18, 24], "batchnorm": [14, 70, 71], "mathemat": [14, 18, 70, 71], "w_": 14, "b_": 14, "w": [14, 67], "layernormpr": [14, 26, 29], "eff": 14, "ext": 14, "wise": [14, 36], "computation": [14, 71], "wish": 14, "defin": [14, 18, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 69, 70, 71], "x_1": [14, 71], "x_0": [14, 71], "x_2": [14, 71], "frac": [14, 71], "x_3": 14, "cdot": 14, "x_4": 14, "preced": [14, 69, 70, 71], "never": [14, 71], "w_write": 14, "keepdim": 14, "fed": [14, 36], "1000": [14, 35, 69, 71], "recreat": 14, "onto": [14, 38, 70], "By": [14, 35, 37, 38, 40, 69, 70, 71], "mix": [14, 69, 70, 71], "linearli": 14, "technic": [14, 70, 71], "deriv": [14, 71], "broadcast_b_v": 14, "broadcast": 14, "And": [14, 40, 70, 71], "destination_posit": [14, 71], "source_posit": [14, 71], "source_": 14, "destin": [14, 15, 40, 71], "behavior": [14, 15, 38, 70], "resolut": [14, 38], "cfg_dict": [14, 38], "cache_dir": [14, 69], "torch_dtyp": 14, "bfloat16": 14, "boolean": [14, 37, 40, 69, 70, 71], "max_new_token": [14, 71], "stop_at_eo": 14, "eos_token_id": [14, 69], "do_sampl": 14, "top_k": [14, 69, 70, 71], "top_p": [14, 69], "temperatur": [14, 69, 71], "freq_penalti": [14, 69], "use_past_kv_cach": 14, "verbos": 14, "pos_plus_new_token": 14, "eos_token": 14, "reach": [14, 71], "avoid": [14, 15, 39, 69, 70, 71], "fiddl": 14, "rag": 14, "eot": 14, "throw": 14, "enter": [14, 70, 71, 73], "messi": [14, 71], "maximum": [14, 15, 18, 63, 71], "stable_lm": 14, "distribut": [14, 68, 69, 70, 71], "greedi": [14, 69], "search": [14, 36, 70, 71], "max": [14, 70], "mass": 14, "random": [14, 15, 35, 63, 70, 71], "temp": [14, 69], "inf": 14, "uniform": [14, 69], "frequenc": [14, 15, 69, 70], "penalti": [14, 69], "penalis": 14, "speed": [14, 70], "applic": [14, 15, 69], "whatev": [14, 70], "tqdm": [14, 71], "get_token_posit": [14, 70, 71], "single_token": [14, 71], "present": 14, "gotcha": [14, 16, 70], "Be": 14, "care": [14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 70, 71], "weird": [14, 15, 70, 71], "carefulli": [14, 70], "correspond": [14, 24, 36, 40, 69, 70, 71], "dummi": [14, 37, 71], "init_weight": [14, 15], "empti": [14, 37], "bulk": 14, "seed": [14, 15, 63, 71], "determin": [14, 18, 40, 68, 69, 70, 71], "NOT": [14, 37, 69, 71], "scheme": 14, "tell": [14, 35, 70, 71], "round": [14, 35, 70, 71], "18182": 14, "fan_in": [14, 69], "tha": 14, "kaim": [14, 69], "despit": [14, 71], "xavier": [14, 69], "fan_out": 14, "transformerencod": 14, "exact": 14, "72253": 14, "mup": [14, 15], "haven": 14, "2203": 14, "03466": 14, "input_to_emb": 14, "special": [14, 71], "redwood": [14, 70, 71], "load_sample_training_dataset": 14, "dataset": [14, 35, 63, 69, 71], "10k": [14, 35, 69], "get_dataset": [14, 69], "appropri": [14, 71], "info": [14, 15, 40, 69, 71], "download": [14, 69, 71], "locat": [14, 40, 70, 71], "pt": 14, "openwebtext": [14, 35, 69], "karma": [14, 35], "reddit": [14, 35], "pile": [14, 35, 38, 69, 71, 72], "imperfectli": 14, "suppli": 14, "valid": [14, 35, 70], "loss_fn": [14, 71], "per_token": [14, 69, 71], "lm_cross_entropy_loss": [14, 69], "move_model_modules_to_devic": 14, "process_weights_": 14, "cleaner": 14, "experiment": 14, "argu": [14, 71], "somewhat": [14, 70, 71], "w_qk": [14, 18, 71], "w_ov": [14, 18, 71], "hopefulli": [14, 73], "column": [14, 40, 69], "rotat": [14, 15, 18, 71], "nth": 14, "formula": 14, "r": 14, "refactor": 14, "diagon": [14, 70, 71], "asymmetri": 14, "fiddli": 14, "preserv": [14, 70, 71], "too": [14, 40, 70], "bilinear": [14, 71], "dimension": [14, 15], "coordin": 14, "sample_datapoint": 14, "implicitli": [14, 40, 71], "hasn": 14, "manual": [14, 69, 71], "replac": [14, 15, 40, 70, 71, 73], "choic": [14, 70], "truncat": [14, 35, 69, 71], "set_token": [14, 15], "pretrainedtoken": 14, "set_ungroup_grouped_query_attent": 14, "ungroup_grouped_query_attent": [14, 15], "ungroup": [14, 15], "gqa": 14, "set_use_attn_in": 14, "use_attn_in": [14, 15], "set_use_attn_result": 14, "easili": [14, 69, 70, 71], "burn": 14, "set_use_hook_mlp_in": 14, "use_hook_mlp_in": [14, 15], "set_use_split_qkv_input": 14, "use_split_qkv_input": [14, 15], "to_single_str_token": 14, "int_token": 14, "uncertain": 14, "to_token": [14, 69, 70, 71], "to_str_token": [14, 16, 70, 71], "weirdli": [14, 70, 71], "gotcha2": 14, "letter": [14, 71], "capit": [14, 70, 71], "shoot": [14, 71], "gotcha3": 14, "exce": 14, "str_token": [14, 70], "to_str": [14, 70, 71], "numpi": [14, 15, 69, 70], "arrai": [14, 16, 69], "long": [14, 71], "window": [14, 15, 69], "tokens_to_residual_direct": [14, 70], "mislead": [14, 70], "integ": [14, 69, 70, 71], "residual_direct": 14, "namedtupl": 14, "dataclass": [15, 37], "act_fn": [15, 72], "ep": 15, "1e": [15, 38], "05": [15, 38], "use_attn_scal": 15, "attn_scal": 15, "use_local_attn": 15, "original_architectur": 15, "from_checkpoint": 15, "checkpoint_label_typ": [15, 71], "window_s": [15, 18, 19, 24], "attn_typ": [15, 18, 19, 24, 30], "init_mod": 15, "normalization_typ": 15, "attention_dir": 15, "attn_onli": [15, 72], "initializer_rang": 15, "scale_attn_by_inverse_layer_idx": 15, "final_rm": 15, "d_vocab_out": [15, 34], "parallel_attn_mlp": 15, "rotary_dim": [15, 18], "n_param": [15, 72], "use_hook_token": 15, "gated_mlp": 15, "tokenizer_prepends_bo": 15, "n_key_value_head": [15, 24, 72], "post_embedding_ln": 15, "rotary_bas": 15, "10000": [15, 18, 71], "trust_remote_cod": 15, "rotary_adjacent_pair": 15, "load_in_4bit": 15, "num_expert": 15, "experts_per_token": 15, "relative_attention_max_dist": 15, "relative_attention_num_bucket": 15, "decoder_start_token_id": 15, "tie_word_embed": 15, "use_normalization_before_and_aft": 15, "attn_scores_soft_cap": 15, "output_logits_soft_cap": 15, "use_ntk_by_parts_rop": 15, "ntk_by_parts_low_freq_factor": 15, "ntk_by_parts_high_freq_factor": 15, "ntk_by_parts_factor": 15, "8": [15, 18, 35, 36, 70, 71, 72], "AND": 15, "feedforward": 15, "network": [15, 70, 71], "vocab": 15, "lowercas": 15, "relu": [15, 69, 72], "gelu": [15, 38, 71, 72], "silu": [15, 72], "gelu_new": [15, 69], "gelu_fast": [15, 69], "epsilon": 15, "5": [15, 18, 35, 36, 38, 40, 69, 70, 71, 72], "THEN": 15, "intens": 15, "divid": [15, 36, 69, 70], "distanc": [15, 18, 70], "xavier_uniform": 15, "xavier_norm": 15, "kaiming_uniform": 15, "kaiming_norm": 15, "rmspre": 15, "pipelin": 15, "parallel": [15, 69, 70], "aka": 15, "unidirect": 15, "bidirect": [15, 71], "gain": [15, 69], "layer_id": [15, 18, 19, 24, 30], "numer": [15, 16, 18, 19, 24, 71], "stabil": [15, 18, 19, 24, 71], "fp16": 15, "rotari": [15, 18], "blog": [15, 18], "eleuth": [15, 18, 69, 71], "res_stream": 15, "sinusoid": 15, "dumb": 15, "mainli": 15, "curs": 15, "init": 15, "law": 15, "pdf": [15, 18, 32, 35], "2001": 15, "08361": 15, "Will": [15, 40], "interven": [15, 37, 40, 70], "add_bos_token": [15, 69], "bitsandbyt": 15, "moe": 15, "bucket": 15, "tie": 15, "word": [15, 69, 70, 71], "gemma": [15, 38, 42, 72], "softcap": 15, "soft_cap": 15, "tanh": 15, "squash": 15, "interv": [15, 36], "ntk": 15, "adjust": [15, 70, 71], "interpol": 15, "2309": 15, "00071": 15, "threshold": 15, "high": [15, 69, 70, 71], "rate": [15, 63, 71], "strategi": [15, 71], "from_dict": 15, "config_dict": 15, "is_layer_norm_activ": 15, "set_seed_everywher": 15, "to_dict": 15, "unwrap": [15, 69], "duplic": [15, 36, 70, 71], "get_singular_vector": 16, "vector_typ": 16, "layer_index": [16, 70], "num_vector": 16, "plot": [16, 71], "pysvelt": [16, 71], "instabl": 16, "d": [16, 35, 36, 38, 70, 72], "medium": [16, 38, 72], "svd_interpret": 16, "22": [16, 35, 69, 70, 71], "all_token": 16, "np": [16, 69, 70], "def": [16, 70, 71], "plot_matrix": 16, "filter": [16, 37, 38, 69, 71], "topk": [16, 70], "topktabl": 16, "obj_typ": 16, "abstract_attent": 17, "bert_block": 17, "bert_emb": 17, "bert_mlm_head": 17, "grouped_query_attent": 17, "layer_norm": [17, 70], "layer_norm_pr": 17, "rms_norm": 17, "rms_norm_pr": 17, "t5_attent": 17, "t5_block": 17, "token_typed_emb": 17, "transformer_block": 17, "abstractattent": [18, 19, 24, 30], "abc": [18, 71], "pure": 18, "glossari": 18, "sorri": 18, "underli": [18, 40, 70, 71], "destination_residu": 18, "destination_po": 18, "source_po": [18, 71], "abstract": [18, 70, 71], "groupedqueryattent": [18, 24], "enforc": 18, "child": 18, "better_abc": 18, "abstract_attribut": 18, "stackoverflow": 18, "question": [18, 70, 71], "23831510": 18, "256": [18, 19, 24, 71, 72], "alibi": 18, "apply_causal_mask": 18, "pos_plus_past_kv_pos_offset": 18, "past_kv_pos_offset": [18, 27, 69], "offset_po": [18, 27, 33, 69], "apply_rotari": 18, "calculate_attention_scor": [18, 24], "calculate_qkv_matric": [18, 24], "query_input": [18, 24], "key_input": [18, 24], "kv_po": [18, 30, 31], "value_input": [18, 24], "calculate_sin_cos_rotari": 18, "sine": 18, "cosin": 18, "wave": 18, "inexplic": 18, "adjac": [18, 70], "neox": [18, 38, 42, 71, 72], "clue": [18, 70], "resolv": 18, "calculate_z_scor": [18, 24], "static": [18, 35], "create_alibi_bia": 18, "head_idx": 18, "2108": 18, "12409": 18, "broad": [18, 70], "proport": [18, 69], "distant": 18, "0000": [18, 70], "0625": 18, "1250": 18, "1875": 18, "0039": 18, "0078": 18, "0117": 18, "create_alibi_multipli": 18, "geometr": 18, "ratio": [18, 69, 70, 71], "16": [18, 69, 70, 71, 72], "5000": 18, "2500": [18, 70], "0312": 18, "0156": 18, "7071": 18, "3536": 18, "1768": 18, "0884": 18, "0442": 18, "0221": 18, "0110": 18, "0055": 18, "create_alibi_slop": 18, "slope": 18, "triangl": 18, "lower": [18, 35, 36, 69, 70, 71], "bottom": [18, 71], "corner": 18, "kv_head_index": [18, 24], "past_kv_cache_entri": [18, 31, 33], "hookedtransformerkeyvaluecacheentri": [18, 31, 33, 39], "additive_attention_mask": [18, 20, 31], "position_bia": [18, 30, 31], "irrelev": [18, 70, 71], "past": [18, 39, 70], "rotate_every_two": 18, "x0": 18, "x1": 18, "param": [19, 63, 69, 71], "convent": [19, 69, 70, 71], "mistal": [19, 24], "bertblock": 20, "transformerblock": [20, 33], "except": [20, 70, 71], "overridden": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 69], "subclass": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "recip": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "afterward": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "former": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "regist": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "latter": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 71], "silent": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "bertemb": 21, "input_id": 21, "bertmlmhead": 22, "purpos": [22, 35, 70, 71], "resid": 22, "2305": 24, "13245": 24, "hood": 24, "_w_k": 24, "_w_v": 24, "getter": 24, "similarli": 24, "kept": 24, "repeat_interleav": 24, "unexpand": 24, "expan": 24, "n_query_head": 24, "gpa": 24, "normalis": [26, 70], "posemb": 27, "root": [28, 29, 71], "rmsnormpr": 29, "t5attent": [30, 31], "has_relative_attention_bia": 30, "expect": [30, 35, 36, 70, 71], "precomput": 30, "feed": [30, 35, 69, 71], "positional_bia": 30, "compute_relative_attention_bia": 30, "query_len": 30, "key_len": 30, "query_length": 30, "key_length": 30, "bin": 30, "t5block": 31, "block_index": [31, 33], "is_decod": 31, "t5layernorm": 31, "inst": 31, "encoder_additive_attention_mask": 31, "encoder_po": 31, "encoder_hidden_st": 31, "_description_": 31, "_type_": [31, 37], "tokentypeemb": 32, "1810": 32, "04805": 32, "apply_mlp": 33, "normalized_resid": 33, "central": [33, 70], "positional_embeddings_typ": 33, "canbeusedasmlp": 33, "evalu": [35, 37, 70, 71], "rough": [35, 71], "cheapli": 35, "roughli": [35, 70, 71], "baselin": 35, "ioidataset": 35, "noun": 35, "num_sampl": 35, "symmetr": 35, "ioi_ev": 35, "476": 35, "met": 35, "alic": 35, "bob": 35, "charli": 35, "ball": [35, 70], "book": 35, "397": 35, "get_default_nam": 35, "get_default_noun": 35, "get_default_templ": 35, "get_sampl": 35, "evaluate_on_dataset": 35, "data_load": 35, "induction_loss": [35, 71], "subseq_len": 35, "384": [35, 71], "io": [35, 36, 70, 71], "accuraci": [35, 36, 69], "make_code_data_load": 35, "codeparrot": [35, 69], "dump": 35, "presum": [35, 70], "natur": [35, 70, 71], "make_owt_data_load": 35, "corpu": [35, 69], "make_pile_data_load": 35, "eleutherai": [35, 38], "english": [35, 71, 73], "academ": 35, "internet": [35, 71], "make_wiki_data_load": 35, "wikitext": 35, "wikipedia": [35, 69, 71], "articl": [35, 69, 70, 71], "bother": 35, "quarantin": 35, "nowadai": 35, "leakag": 35, "though": [35, 69, 70, 71], "sanity_check": 35, "paragraph": [35, 71], "zoom": [35, 40, 70], "quick": [35, 36, 71], "saniti": [35, 70], "ok": [35, 70, 71], "gone": [35, 70, 71], "wrong": [35, 37, 70], "compute_head_attention_similarity_scor": 36, "attention_pattern": [36, 71], "detection_pattern": 36, "exclude_bo": 36, "exclude_current_token": 36, "error_measur": 36, "mul": 36, "exclude_bcurrent_token": 36, "detect_head": 36, "seq": [36, 69], "previous_token_head": 36, "duplicate_token_head": 36, "induction_head": 36, "headnam": 36, "straightforward": [36, 70], "fraction": 36, "alloc": 36, "prohibit": [36, 71], "cours": [36, 70], "raw": [36, 70], "perfect": [36, 70], "examin": 36, "switch": 36, "advantag": 36, "closer": 36, "head_nam": 36, "ntensor": 36, "ioi": [36, 70, 71], "spacifi": 36, "analyz": 36, "paid": [36, 70, 71], "get_duplicate_token_head_detection_pattern": 36, "dynalist": 36, "n2zwtnoyhru1s4vnfsaq519j": 36, "2ukvedzonghl5uhugvhroxeo": 36, "get_induction_head_detection_pattern": 36, "_tfvup5csv5orithmqwj0gsi": 36, "get_previous_token_head_detection_pattern": 36, "0o5vohe9xezn8ertywkh7ioc": 36, "get_supported_head": 36, "hookfunct": 37, "_hookfunctionprotocol": 37, "inspir": [37, 71, 73], "garcon": [37, 71, 73], "ident": [37, 69, 70, 71], "wrap": [37, 71], "add_hook": [37, 70], "bwd": 37, "fn": 37, "hook_nam": 37, "add_perma_hook": [37, 71], "clear_context": 37, "remove_hook": 37, "including_perman": 37, "interfac": [37, 71, 73], "nice": [37, 70], "run_with_hook": [37, 70, 71], "temporari": [37, 69, 71], "debug": [37, 38, 63], "intent": 37, "reset_hook": [37, 71], "goe": [37, 70, 71], "reset_hooks_end": [37, 70], "add_caching_hook": 37, "names_filt": [37, 70, 71], "callabl": [37, 40], "incl_bwd": 37, "namesfilt": 37, "lambda": [37, 70, 71], "cache_som": 37, "check_and_add_hook": 37, "get_caching_hook": 37, "fwd_hook": [37, 70, 71], "bwd_hook": 37, "hook_dict": 37, "exit": [37, 69], "clear": [37, 71], "reset": 37, "my_hook": 37, "hooked_loss": 37, "mod_dict": 37, "remove_all_hook_fn": 37, "model_kwarg": 37, "degrad": 37, "lenshandl": 37, "removablehandl": 37, "context_level": 37, "hold": 37, "perman": 37, "hug": 38, "face": 38, "hub": [38, 69], "768": [38, 70, 71, 72], "layer_norm_ep": 38, "init_rang": 38, "02": 38, "1024": [38, 69, 71, 72], "3072": [38, 71, 72], "12": [38, 70, 71, 72], "model_alias": 38, "01": 38, "yi": [38, 72], "34b": [38, 72], "6b": [38, 71, 72], "arthurconmi": 38, "redwood_attn_2l": [38, 72], "baidicoot": 38, "3b": [38, 71, 72], "125m": [38, 71, 72], "20b": [38, 71, 72], "pythia": [38, 72], "4b": [38, 72], "dedup": [38, 72], "12b": [38, 72], "13b": [38, 71, 72], "14m": [38, 72], "160m": [38, 72], "seed1": [38, 72], "seed2": [38, 72], "seed3": [38, 72], "1b": [38, 72], "800m": 38, "8b": [38, 71, 72], "31m": [38, 72], "410m": [38, 72], "350m": 38, "9b": [38, 72], "70m": [38, 72], "19m": [38, 72], "2l512w": 38, "lr": [38, 63], "attn_only_1l512w_c4_cod": 38, "c4": [38, 69, 71], "attn_only_2l512w_c4_cod": 38, "attn_only_3l512w_c4_cod": 38, "attn_only_4l512w_c4_cod": 38, "gelu_1l512w_c4_cod": 38, "gelu_2l512w_c4_cod": 38, "gelu_3l512w_c4_cod": 38, "gelu_4l512w_c4_cod": 38, "solu_10l1280w_c4_cod": 38, "10l": [38, 71, 72], "solu_10l_v22_old": 38, "solu_12l1536w_c4_cod": 38, "12l": [38, 71, 72], "solu_12l_v23_old": 38, "solu_1l512w_c4_cod": 38, "solu_1l512w_wiki_finetun": 38, "wiki": [38, 69, 70, 71, 72], "finetun": 38, "solu_1l_v9_old": 38, "solu_2l512w_c4_cod": 38, "solu_2l_v10_old": 38, "solu_3l512w_c4_cod": 38, "solu_4l512w_c4_cod": 38, "solu_4l512w_wiki_finetun": 38, "solu_4l_v11_old": 38, "solu_6l768w_c4_cod": 38, "6l": [38, 71, 72], "solu_6l_v13_old": 38, "solu_8l1024w_c4_cod": 38, "8l": [38, 71, 72], "solu_8l_v21_old": 38, "qwen": [38, 42, 61, 72], "14b": [38, 72], "1_8b": 38, "qwen1": [38, 72], "5b": [38, 71, 72], "forev": 38, "mgpt": [38, 72], "bigcod": 38, "santacod": [38, 72], "bigscienc": 38, "1b1": [38, 72], "1b7": [38, 72], "560m": [38, 72], "7b1": [38, 72], "codellama": [38, 72], "hf": 38, "codellamallama": [38, 72], "distilgpt2": [38, 71], "distillgpt2": [38, 72], "distil": [38, 71], "facebook": 38, "xxl": 38, "30b": [38, 71, 72], "xxxl": 38, "xl": [38, 71, 72], "66b": [38, 71, 72], "xxxxl": 38, "27b": [38, 72], "2b": [38, 72], "65b": [38, 72], "70b": [38, 72], "microsoft": 38, "phi": [38, 42, 59, 72], "mini": 38, "4k": 38, "1_5": [38, 72], "nemo": [38, 72], "2407": [38, 72], "roneneldan": 38, "tinystori": 38, "1layer": 38, "21m": [38, 72], "28m": [38, 72], "2layer": 38, "33m": [38, 72], "3m": [38, 72], "8m": [38, 72], "instuct": 38, "stabilityai": 38, "stablelm": [38, 71, 72], "alpha": [38, 72], "x21": 38, "arwen": 38, "battlestar": 38, "x49": 38, "beren": 38, "caprica": 38, "x81": 38, "celebrimbor": 38, "darkmatt": 38, "x343": 38, "durin": 38, "eowyn": 38, "x777": 38, "expans": 38, "alias": 38, "non_hf_hosted_model_nam": 38, "qwen2": [38, 42, 72], "32b": [38, 72], "72b": [38, 72], "qwq": [38, 72], "preview": [38, 72], "get_checkpoint_label": [38, 71], "label_typ": 38, "get_num_params_of_pretrain": 38, "suffici": [38, 70], "get_pretrained_model_config": 38, "hf_cfg": 38, "automodel": 38, "autoconfig": 38, "infrastructur": [38, 70, 71, 73], "ourselv": [39, 69, 71, 73], "previous_attention_mask": 39, "pos_so_far": 39, "append": [39, 70, 71], "prefix": 39, "append_attention_mask": 39, "new_token": 39, "freez": 39, "init_cach": 39, "unfreez": 39, "past_kei": 39, "jaxtyp": [39, 70, 71], "past_valu": 39, "new_kei": 39, "new_valu": 39, "init_cache_entri": 39, "structur": [40, 71], "generic_activation_patch": 40, "specialis": [40, 70], "introduc": [40, 70], "rome": [40, 70, 71], "baulab": 40, "corrupt": [40, 70, 71], "iter": [40, 69, 70, 71], "localis": [40, 70, 71], "__from__": 40, "__to": 40, "__the": 40, "confid": [40, 70, 71], "intuit": [40, 70, 71], "diffus": [40, 70], "spread": [40, 70], "connect": [40, 70], "ultim": [40, 70], "tend": [40, 71], "extrem": [40, 70, 71, 73], "eiffel": 40, "tower": 40, "pari": 40, "factual": [40, 70], "recal": [40, 70], "colosseum": 40, "corrupted_token": [40, 70, 71], "clean_cach": [40, 70, 71], "patching_metr": 40, "patch_sett": 40, "index_axis_nam": 40, "src_po": [40, 70], "dest_po": [40, 70, 71], "index_df": 40, "datafram": 40, "return_index_df": 40, "counterfactu": [40, 70, 71], "Then": 40, "index_to_act_nam": 40, "recov": [40, 70, 71], "diff": [40, 70], "corrupted_activ": 40, "chunk": 40, "fill": 40, "flatten": [40, 70, 71], "patched_output": 40, "get_act_patch_attn_head_all_pos_everi": 40, "patch_typ": 40, "get_act_patch_attn_head_by_pos_everi": 40, "get_act_patch_attn_head_k_all_po": 40, "corruptedactiv": 40, "patchedactiv": 40, "layer_head_vector_patch_sett": 40, "axisnam": 40, "get_act_patch_attn_head_k_by_po": 40, "layer_pos_head_vector_patch_sett": 40, "get_act_patch_attn_head_out_all_po": 40, "get_act_patch_attn_head_out_by_po": 40, "get_act_patch_attn_head_pattern_all_po": 40, "layer_head_pattern_patch_sett": 40, "get_act_patch_attn_head_pattern_by_po": 40, "layer_head_pos_pattern_patch_sett": 40, "get_act_patch_attn_head_pattern_dest_src_po": 40, "layer_head_dest_src_pos_pattern_patch_sett": 40, "get_act_patch_attn_head_q_all_po": 40, "get_act_patch_attn_head_q_by_po": 40, "get_act_patch_attn_head_v_all_po": 40, "get_act_patch_attn_head_v_by_po": 40, "get_act_patch_attn_out": 40, "layer_pos_patch_sett": 40, "get_act_patch_block_everi": 40, "get_act_patch_mlp_out": 40, "get_act_patch_resid_mid": 40, "get_act_patch_resid_pr": 40, "clean_activ": 40, "weight_convers": 41, "coder": 42, "mingpt": 42, "nanogpt": 42, "neel_solu_old": 42, "phi3": 42, "convert_bloom_weight": 44, "convert_coder_weight": 45, "convert_mistral_weight": 51, "convert_mixtral_weight": 52, "convert_phi_weight": 58, "convert_phi3_weight": 59, "convert_qwen_weight": 60, "convert_qwen2_weight": 61, "convert_t5_weight": 62, "hookedtransformertrainconfig": 63, "num_epoch": 63, "001": 63, "max_grad_norm": 63, "weight_decai": 63, "optimizer_nam": 63, "adam": 63, "warmup_step": 63, "save_everi": 63, "save_dir": 63, "wandb": 63, "wandb_project_nam": 63, "print_everi": 63, "max_step": 63, "hyperparamet": [63, 69], "epoch": 63, "decai": 63, "warmup": 63, "wandb_project": 63, "termin": 63, "activation_funct": 64, "addmm": 64, "batch_addmm": 66, "d_out": [66, 69], "d_in": [66, 69], "fuse": 66, "conv1d": 66, "9ba9369a2557e53a01378199a9839ec6e82d8bc7": 66, "src": 66, "pytorch_util": 66, "l102": 66, "l106": 66, "vanilla_addmm": 66, "mat1": 66, "mat2": 66, "typecheck": 66, "complex_attn_linear": 67, "almost": [67, 70], "simple_attn_linear": 67, "extra": [67, 69], "assist": 68, "get_device_for_block_index": 68, "target": 68, "move_to_and_update_config": 68, "vari": [69, 70], "throughout": [69, 71], "locallyoverridendefault": 69, "restor": 69, "overriden": 69, "input_slic": 69, "syntax": [69, 70, 71], "reduc": [69, 70, 71], "leav": [69, 71], "elif": 69, "1d": 69, "sliceinput": 69, "valueerror": 69, "abov": [69, 70, 71], "max_ctx": 69, "int32": 69, "slice_input": 69, "calc_fan_in_and_fan_out": 69, "fan": 69, "composition_scor": 69, "broadcast_dim": 69, "leading_dims_left_and_right": 69, "download_file_from_hf": 69, "repo_nam": 69, "file_nam": 69, "subfold": 69, "home": 69, "runner": 69, "force_is_torch": 69, "json": 69, "pth": 69, "extens": [69, 70], "layer_typ": [69, 70], "shorthand": 69, "loop": [69, 70, 71, 73], "hack": [69, 71], "stuff": [69, 71], "readabl": 69, "digit": [69, 71], "k6": 69, "scale4ln1": 69, "appear": [69, 71], "distinguish": [69, 70], "hook_k": [69, 71], "hook_pr": [69, 71], "27": [69, 70, 71], "hook_norm": [69, 71], "pre5": 69, "get_attention_mask": 69, "leftmost": 69, "rightmost": 69, "consid": 69, "get_cumsum_along_dim": 69, "dataset_nam": 69, "000": [69, 71], "enorm": [69, 71], "100gb": 69, "2tb": 69, "effort": [69, 70], "dataload": 69, "fanci": 69, "data_dir": 69, "approx": [69, 70, 71], "ton": [69, 73], "divers": [69, 70, 71], "coloss": 69, "crawl": 69, "bigger": 69, "c4_code": 69, "friendli": 69, "22m": [69, 71], "5m": 69, "20220301": 69, "en": [69, 71], "get_devic": [69, 70, 71], "get_input_with_manually_prepended_bo": 69, "autotoken": 69, "get_nested_attr": 69, "obj": 69, "attr_str": 69, "retriev": 69, "nest": 69, "hierarchi": 69, "get_offset_position_id": 69, "offset": [69, 70, 71], "get_tokenizer_with_bo": 69, "Such": [69, 70], "llamatoken": 69, "get_tokens_with_bos_remov": 69, "init_kaiming_normal_": 69, "nonlinear": 69, "std": 69, "init_kaiming_uniform_": 69, "init_xavier_normal_": 69, "init_xavier_uniform_": 69, "is_lower_triangular": 69, "is_squar": 69, "keep_single_column": 69, "col_nam": 69, "lm_accuraci": 69, "seq_len": [69, 70, 71], "altern": 69, "override_or_use_default_valu": 69, "default_flag": 69, "print_gpu_mem": 69, "step_nam": 69, "repeat_along_head_dimens": 69, "clone_tensor": 69, "sample_logit": 69, "final_logit": [69, 70], "vocab_s": 69, "argmaxi": 69, "90": 69, "renormalis": 69, "mutual": 69, "neither": [69, 70], "input_token": 69, "todo": 69, "edg": 69, "randn": [69, 71], "uniqu": 69, "return_count": 69, "set_nested_attr": 69, "prepend_space_to_answ": 69, "eleph": 69, "endoftext": [69, 70, 71], "14": [69, 70, 71, 72], "51": [69, 71], "0th": [69, 70], "59": [69, 71, 72], "ground": [69, 70], "1th": [69, 70], "41": [69, 71], "tree": 69, "2th": [69, 70], "3th": [69, 70], "45": [69, 71], "car": 69, "4th": [69, 70], "13": [69, 70, 71], "92": [69, 70], "55": [69, 70, 71], "river": 69, "5th": [69, 70], "79": 69, "25": [69, 70, 71, 72], "street": 69, "6th": [69, 70], "77": 69, "21": [69, 70, 71], "7th": [69, 70], "75": 69, "hill": 69, "8th": [69, 70], "swing": 69, "9th": [69, 70], "46": [69, 71, 72], "61": [69, 71, 72], "park": [69, 70], "to_numpi": [69, 70, 71], "tokenize_and_concaten": 69, "max_length": 69, "column_nam": 69, "num_proc": 69, "eo": [69, 71], "reshap": [69, 70], "____": 69, "drop": [69, 71], "faster": [69, 70, 71], "parallelis": [69, 71], "chop": 69, "privileg": 69, "earli": [69, 71], "cnn": [69, 71], "bos_token_id": 69, "swap": [69, 70], "runtim": [70, 71], "hardwar": [70, 71], "pane": [70, 71], "sidebar": [70, 71], "navig": [70, 71], "vscode": [70, 71], "outlin": 70, "tab": 70, "dropdown": [70, 71], "arrow": [70, 71], "page": [70, 71], "ctrl": [70, 71], "in_colab": [70, 71], "circuitsvi": [70, 71], "node": [70, 71], "curl": [70, 71], "fssl": [70, 71], "deb": [70, 71], "nodesourc": [70, 71], "setup_16": [70, 71], "sudo": [70, 71], "bash": [70, 71], "apt": [70, 71], "nodej": [70, 71], "noqa": [70, 71], "ipython": [70, 71], "get_ipython": [70, 71], "ip": [70, 71], "extension_manag": [70, 71], "autoreload": [70, 71], "functool": [70, 71], "plotli": [70, 71], "express": [70, 71], "px": [70, 71], "pio": [70, 71], "attention_head": 70, "fancy_einsum": [70, 71], "ifram": 70, "differenti": [70, 71], "simplic": 70, "imshow": [70, 71], "color_continuous_midpoint": [70, 71], "color_continuous_scal": [70, 71], "rdbu": [70, 71], "scatter": [70, 71], "xaxi": [70, 71], "yaxi": [70, 71], "caxi": [70, 71], "color": [70, 71], "principl": [70, 71, 73], "fun": [70, 71, 73], "gap": [70, 71, 73], "plai": [70, 71, 73], "flow": [70, 71, 73], "toolkit": [70, 71], "stylist": 70, "slowli": 70, "convei": 70, "tag": 70, "asid": 70, "flavour": 70, "weed": 70, "star": 70, "tagexampl": 70, "capabl": [70, 71], "interview": [70, 71], "kevin": [70, 71], "wang": 70, "twitter": 70, "thread": 70, "overview": 70, "bottl": [70, 71], "milk": [70, 71], "26": [70, 71, 72], "Their": 70, "skimp": 70, "rigour": 70, "suggest": 70, "evid": 70, "80m": [70, 71], "simplif": 70, "nbval_ignore_output": [70, 71], "stabl": 70, "example_prompt": 70, "example_answ": 70, "39": [70, 71], "lt": [70, 71], "gt": [70, 71], "09": [70, 71], "70": 70, "07": [70, 71], "38": [70, 71], "67": 70, "35": [70, 71], "54": [70, 71], "11": [70, 71, 72], "84": [70, 71], "73": 70, "hi": [70, 71], "06": 70, "her": [70, 71], "74": 70, "52": [70, 71, 72], "49": [70, 71], "jesu": 70, "97": 70, "42": [70, 71, 72], "him": 70, "subword": 70, "frequent": 70, "substr": [70, 71], "headach": 70, "annoi": [70, 71], "devot": 70, "sensibl": 70, "later": [70, 71], "wherev": 70, "flesh": 70, "prompt_format": 70, "jame": 70, "dan": 70, "sid": 70, "appl": 70, "martin": 70, "ami": 70, "drink": 70, "correct_token": 70, "insert": 70, "filler": 70, "newlin": 70, "intellig": 70, "complic": 70, "aggreg": 70, "original_logit": 70, "upon": 70, "subject": [70, 71], "logits_to_ave_logit_diff": 70, "per_prompt": 70, "answer_logit": 70, "gather": 70, "answer_logit_diff": 70, "detach": [70, 71], "decim": [70, 71], "original_average_logit_diff": 70, "3370": 70, "2020": 70, "7090": 70, "7970": 70, "7200": 70, "2810": 70, "6010": 70, "7670": 70, "552": 70, "33": [70, 71], "dive": 70, "spend": [70, 71], "engag": 70, "decent": [70, 71], "hypothes": 70, "cheat": [70, 71], "hypothesi": 70, "scienc": 70, "belief": 70, "trap": 70, "flounder": 70, "dogmat": 70, "overconfid": 70, "unwil": 70, "contradict": 70, "flinch": 70, "disconfirm": 70, "focu": 70, "primit": 70, "nearbi": 70, "came": 70, "trigram": 70, "symmetri": 70, "cancel": 70, "inhibit": 70, "spoiler": 70, "simplist": 70, "importantli": [70, 71], "perfectli": [70, 71], "final_residual_stream": 70, "eleg": 70, "particularli": 70, "aspect": 70, "nicer": 70, "inde": 70, "log_prob": 70, "log_softmax": 70, "logsumexp": 70, "decid": 70, "pronoun": 70, "refin": 70, "friendlier": 70, "answer_residual_direct": 70, "logit_diff_direct": 70, "account": 70, "w_u_fold": 70, "unigram": [70, 71], "statist": [70, 71], "opposit": 70, "hook_normalis": 70, "sub_layer_typ": 70, "final_token_residual_stream": 70, "scaled_final_token_residual_stream": 70, "average_logit_diff": 70, "residual_stack_to_logit_diff": 70, "scaled_residual_stack": 70, "fascinatingli": 70, "utterli": 70, "unabl": 70, "hover": [70, 71], "n_pre": 70, "n_mid": 70, "n_post": 70, "middl": [70, 71], "accumulated_residu": 70, "logit_lens_logit_diff": 70, "arang": 70, "hover_nam": [70, 71], "terminologi": 70, "overload": 70, "kth": 70, "per_layer_residu": 70, "per_layer_logit_diff": 70, "independ": [70, 71, 73], "l9h6": 70, "l9h9": 70, "l10h7": 70, "l11h10": 70, "harm": 70, "strongli": 70, "observ": [70, 71], "144": 70, "claim": 70, "surpris": 70, "7x": 70, "per_head_residu": 70, "per_head_logit_diff": 70, "rearrang": 70, "weren": 70, "alan": [70, 71], "coonei": [70, 71], "illustr": [70, 71], "mistak": 70, "mayb": [70, 71], "sai": [70, 71], "summari": 70, "sole": 70, "visualize_attention_pattern": 70, "local_cach": 70, "local_token": 70, "max_width": 70, "700": 70, "isinst": 70, "batch_index": 70, "combin": [70, 71], "attention_head_nam": 70, "show_cod": 70, "title_html": 70, "br": 70, "div": 70, "width": [70, 71], "top_positive_logit_attr_head": 70, "positive_html": 70, "top_negative_logit_attr_head": 70, "negative_html": 70, "conceptu": 70, "clearli": 70, "compos": [70, 71], "ideal": [70, 71], "david": [70, 71], "bau": [70, 71], "meng": [70, 71], "trace": [70, 71], "anim": 70, "lai": 70, "pro": 70, "con": 70, "Or": 70, "bake": 70, "claus": 70, "tack": 70, "gaussian": 70, "nois": 70, "beforehand": 70, "19": [70, 71], "corrupted_prompt": [70, 71], "corrupted_logit": [70, 71], "corrupted_cach": 70, "corrupted_average_logit_diff": 70, "temporarili": [70, 71], "patch_residual_compon": 70, "corrupted_residual_compon": 70, "normalize_patched_logit_diff": 70, "patched_logit_diff": [70, 71], "wors": [70, 71], "patched_residual_stream_diff": 70, "hook_fn": 70, "patched_logit": [70, 71], "abus": 70, "prompt_position_label": 70, "tok": 70, "_": [70, 71], "enumer": [70, 71], "reus": 70, "patched_attn_diff": 70, "patched_mlp_diff": 70, "patched_attn_logit": 70, "patched_attn_logit_diff": 70, "patched_mlp_logit": 70, "patched_mlp_logit_diff": 70, "late": [70, 71], "contrast": 70, "statement": 70, "mlp0": 70, "destroi": 70, "frame": 70, "unprincipl": 70, "invers": [70, 71], "plausibli": 70, "dedic": 70, "overcom": 70, "love": 70, "someon": 70, "patch_head_vector": 70, "corrupted_head_vector": 70, "patched_head_z_diff": 70, "l8h6": 70, "l8h10": 70, "l7h9": 70, "l5h5": 70, "l6h9": 70, "l3h0": 70, "semi": 70, "disentangl": 70, "familiar": 70, "28": [70, 71, 72], "patched_head_v_diff": 70, "heatmap": 70, "29": [70, 71], "lesson": 70, "head_label": 70, "range_x": 70, "range_i": 70, "31": [70, 71], "patch_head_pattern": 70, "corrupted_head_pattern": 70, "patched_head_attn_diff": 70, "32": [70, 71, 72], "reconsolid": 70, "extend": 70, "l7h3": 70, "specul": 70, "mysteri": [70, 71], "top_heads_by_output_patch": 70, "first_mid_lay": 70, "first_late_lay": 70, "early_head": 70, "mid_head": 70, "logical_and": 70, "late_head": 70, "diagram": [70, 73], "l1h2": 70, "latest": 70, "definit": 70, "priori": 70, "stroke": 70, "didn": 70, "bracket": 70, "serv": [70, 71], "particip": 70, "behav": 70, "l5h0": 70, "wrote": [70, 71, 73], "overkil": 70, "simpler": 70, "repurpos": 70, "machineri": 70, "life": [70, 71], "built": 70, "34": [70, 71], "example_text": [70, 71], "seek": 70, "machin": [70, 71], "example_repeated_text": 70, "example_repeated_token": 70, "example_repeated_logit": 70, "example_repeated_cach": 70, "induction_head_label": 70, "81": 70, "65": 70, "800": 70, "accord": 70, "wildli": 70, "characteris": 70, "superfici": 70, "boost": [70, 71], "anti": 70, "suppress": [70, 71], "pick": [70, 71], "signal": 70, "hook_": 70, "hook_attn": 70, "token_po": 70, "metadata": 70, "36": [70, 71, 72], "prev_token_scor": 70, "prev_token_hook": 70, "dim1": [70, 71], "dim2": [70, 71], "duplicate_token_scor": 70, "duplicate_token_hook": 70, "induction_scor": [70, 71], "induction_hook": 70, "manual_se": [70, 71], "original_token": 70, "randint": [70, 71], "20000": [70, 71], "repeated_token": [70, 71], "pattern_filt": 70, "act_nam": [70, 71], "endswith": [70, 71], "hook_pattern": [70, 71], "0390": 70, "0310": 70, "1890": 70, "1720": 70, "0680": 70, "1570": 70, "0210": 70, "4820": 70, "0030": 70, "1320": 70, "0050": 70, "0020": 70, "0090": 70, "0040": 70, "0010": 70, "instantli": 70, "37": [70, 71], "seen": [70, 71], "mosaic": 70, "40": [70, 71, 72], "fascin": 70, "knock": 70, "naiv": [70, 71], "convers": 70, "flaw": 70, "knockout": 70, "send": 70, "redund": 70, "job": 70, "underestim": 70, "57": [70, 71], "99": [70, 71], "hook_z": [70, 71], "top_name_mov": 70, "top_name_mover_lay": 70, "top_name_mover_head": 70, "ablate_top_head_hook": 70, "ablated_logit": 70, "ablated_cach": 70, "2f": [70, 71], "l10h10": 70, "margin": 70, "obvious": 70, "per_head_ablated_residu": 70, "per_head_ablated_logit_diff": 70, "04": [70, 71], "uniformli": [70, 71], "042": 70, "5200": 70, "4700": 70, "8200": 70, "5100": 70, "2600": 70, "1800": 70, "4300": 70, "5700": 70, "3500": 70, "2900": 70, "6800": 70, "4900": 70, "8700": 70, "4200": 70, "reader": [70, 71], "gentler": 71, "tip": 71, "development_mod": 71, "in_github": 71, "getenv": 71, "github_act": 71, "render": 71, "argh": 71, "notebook_connect": 71, "cv": 71, "hello": 71, "auto": 71, "autograd": 71, "grad_mod": 71, "0x7f027eb64110": 71, "speak": [71, 73], "human": [71, 73], "palm": [71, 73], "nor": [71, 73], "offend": [71, 73], "anthrop": [71, 73], "team": [71, 73], "got": [71, 73], "frustrat": [71, 73], "deepspe": [71, 73], "industri": [71, 73], "heavili": [71, 73], "credit": [71, 73], "nelson": [71, 73], "elhag": [71, 73], "chri": [71, 73], "olah": [71, 73], "model_description_text": 71, "hyper": 71, "1758": 71, "box": 71, "On": 71, "insid": 71, "kinda": 71, "gpt2_cache_no_batch_dim": 71, "gpt2_cach": 71, "gpt2_text": 71, "summar": 71, "supervis": 71, "taskspecif": 71, "gpt2_token": 71, "gpt2_logit": 71, "lock": 71, "grid": 71, "gpt2_str_token": 71, "cell": 71, "attn_hook_nam": 71, "attn_lay": 71, "gpt2_attn_cach": 71, "gpt2_attn": 71, "assert": 71, "neural": 71, "system": 71, "surgic": 71, "surround": 71, "current_activation_valu": 71, "new_activation_valu": 71, "substitut": 71, "relationship": 71, "underr": 71, "janki": 71, "shamelessli": 71, "probepoint": 71, "qualiti": 71, "head_ablation_hook": 71, "layer_to_abl": 71, "head_index_to_abl": 71, "original_loss": 71, "ablated_loss": 71, "3f": 71, "999": 71, "453": 71, "stai": 71, "clean_prompt": 71, "clean_token": 71, "logits_to_logit_diff": 71, "correct_answ": 71, "incorrect_answ": 71, "correct_index": 71, "incorrect_index": 71, "clean_logit": 71, "clean_logit_diff": 71, "corrupted_logit_diff": 71, "276": 71, "738": 71, "residual_stream_patching_hook": 71, "clean_resid_pr": 71, "num_posit": 71, "ioi_patching_result": 71, "temp_hook_fn": 71, "ish": 71, "token_label": 71, "workflow": 71, "michael": 71, "jordan": 71, "surnam": 71, "occurr": 71, "terribl": 71, "halfwai": 71, "input_tensor": 71, "random_token": 71, "repeated_logit": 71, "correct_log_prob": 71, "loss_by_posit": 71, "manipul": 71, "hook_funct": 71, "induction_score_stor": 71, "induction_score_hook": 71, "induction_strip": 71, "pattern_hook_names_filt": 71, "highli": 71, "stripe": 71, "induction_head_lay": 71, "induction_head_index": 71, "single_random_sequ": 71, "repeated_random_sequ": 71, "visualize_pattern_hook": 71, "3d": 71, "four": 71, "300m": 71, "soon": 71, "distilgpt": 71, "distilgpt2_induction_score_stor": 71, "classic": 71, "openai": 71, "85m": [71, 72], "700m": 71, "22b": 71, "300b": 71, "180b": 71, "600": 71, "265": 71, "108m": 71, "bookscorpu": 71, "free": 71, "512": [71, 72], "tractabl": 71, "motif": 71, "80": [71, 72], "shuffl": 71, "scan": 71, "40m": 71, "100m": 71, "200m": 71, "340m": [71, 72], "15b": [71, 72], "13m": [71, 72], "digress": 71, "usefulli": 71, "variengien": 71, "websit": 71, "cleantransformerdemo": 71, "new_activ": 71, "old_activ": 71, "remind": 71, "50267": 71, "named_paramet": 71, "startswith": 71, "fallback": 71, "spam": 71, "dest_posit": 71, "brown": 71, "fox": 71, "lazi": 71, "dog": 71, "num": 71, "print_name_shape_hook_funct": 71, "not_in_late_block_filt": 71, "hook_q": 71, "hook_v": 71, "hook_attn_scor": 71, "hook_attn_out": 71, "hook_resid_mid": 71, "hook_post": 71, "hook_mlp_out": 71, "hook_resid_post": 71, "preconcept": 71, "overhead": 71, "elementwis": 71, "consequ": 71, "rare": 71, "dramat": 71, "degre": 71, "punctuat": 71, "ass": 71, "randomredditor": 71, "unembed_bia": 71, "bias_valu": 71, "bias_indic": 71, "repr": 71, "03": 71, "98": 71, "68": 71, "48": [71, 72], "47": 71, "72": [71, 72], "44": [71, 72], "82": 71, "\u30b5\u30fc\u30c6\u30a3": 71, "83": 71, "x18": 71, "x14": 71, "\u9f8d": 71, "x1b": 71, "x05": 71, "x00": 71, "x06": 71, "x07": 71, "x0c": 71, "x02": 71, "oreandonlin": 71, "x11": 71, "x10": 71, "favour": 71, "6x": 71, "john_bia": 71, "mary_bia": 71, "4f": 71, "exp": 71, "8995": 71, "6034": 71, "6550x": 71, "finit": 71, "invert": 71, "de": 71, "uncommon": 71, "iz": 71, "charact": 71, "example_text_str_token": 71, "example_text_token": 71, "50256": 71, "464": 71, "717": 71, "1517": 71, "345": 71, "761": 71, "284": 71, "3785": 71, "503": 71, "318": 71, "1635": 71, "4919": 71, "1243": 71, "389": 71, "11241": 71, "1143": 71, "4600": 71, "19849": 71, "1462": 71, "62": 71, "2536": 71, "482": 71, "641": 71, "63": 71, "30778": 71, "257": 71, "4731": 71, "656": 71, "262": 71, "16326": 71, "292": 71, "1351": 71, "286": 71, "850": 71, "37336": 71, "25666": 71, "290": 71, "523": 71, "8781": 71, "7301": 71, "644": 71, "2420": 71, "3073": 71, "588": 71, "1675": 71, "10176": 71, "428": 71, "1309": 71, "338": 71, "779": 71, "340": 71, "319": 71, "7322": 71, "signifi": 71, "example_multi_text": 71, "cat": 71, "sat": 71, "mat": 71, "example_multi_text_token": 71, "3797": 71, "3332": 71, "2603": 71, "1107": 71, "1327": 71, "th": 71, "cat_text": 71, "cat_logit": 71, "cat_prob": 71, "capital_the_token_index": 71, "ascii": 71, "squeez": 71, "annoy": 71, "arithmet": 71, "impress": 71, "2342": 71, "2017": 71, "21445": 71, "1000000": 71, "999999": 71, "214": 71, "000000": 71, "9999": 71, "tim": 71, "ne": 71, "el": 71, "messier": 71, "takeawai": 71, "unexpect": 71, "notic": 71, "trip": 71, "confusingli": 71, "forth": 71, "ioi_logits_with_bo": 71, "clair": 71, "mary_logit_with_bo": 71, "claire_logit_with_bo": 71, "ioi_logits_without_bo": 71, "mary_logit_without_bo": 71, "claire_logit_without_bo": 71, "754": 71, "782": 71, "air": 71, "understood": 71, "requisit": 71, "attention_scor": 71, "ab_factor": 71, "9105": 71, "linalg": 71, "eig": 71, "2877e": 71, "00": 71, "8626e": 71, "3121e": 71, "9038e": 71, "08": 71, "1527e": 71, "2877": 71, "3121": 71, "3126e": 71, "3963e": 71, "2029e": 71, "7690e": 71, "2164e": 71, "3126": 71, "3963": 71, "43": 71, "300": 71, "abc_factor": 71, "unfactor": 71, "160": 71, "0830": 71, "ab_unfactor": 71, "isclos": 71, "subspac": 71, "coincid": 71, "negat": 71, "proxi": 71, "lambda_i": 71, "ov_circuit_all_head": 71, "ov_circuit_all_heads_eigenvalu": 71, "complex64": 71, "ov_copying_scor": 71, "zmax": 71, "zmin": 71, "l11h11": 71, "imag": 71, "imaginari": 71, "full_ov_circuit": 71, "full_ov_circuit_eigenvalu": 71, "full_ov_copying_scor": 71, "interestingli": 71, "correl": 71, "outlier": 71, "53": 71, "ansh": 71, "radhakrishnan": 71, "establish": 71, "presid": 71, "barack": 71, "obama": 71, "caught": 71, "embarrass": 71, "scandal": 71, "nthe": 71, "financi": 71, "wife": 71, "chelsea": 71, "she": 71, "woman": 71, "lightweight": 71, "squarethenadd": 71, "hook_squar": 71, "twolayermodel": 71, "layer1": 71, "layer2": 71, "hook_in": 71, "hook_mid": 71, "hook_out": 71, "x_in": 71, "x_mid": 71, "x_out": 71, "model_out": 71, "cache_object": 71, "56": [71, 72], "780": 71, "784": 71, "set_to_zero_hook": 71, "num_checkpoint": 71, "piecewis": 71, "schedul": 71, "crash": 71, "11b": [71, 72], "centr": 71, "hoc": 71, "count": 71, "58": 71, "checkpoint_label": 71, "log_i": 71, "marker": 71, "brief": 71, "suddenli": 71, "500": 71, "visibl": 71, "curv": 71, "briefli": 71, "deliber": 71, "justic": 71, "chosen": 71, "60": [71, 72], "500m": 71, "arbitrarili": 71, "fast": 71, "checkpoint_indic": 71, "checkpointed_model": 71, "tokens_trained_on": 71, "model_for_this_checkpoint": 71, "tokens_seen_for_this_checkpoint": 71, "induction_loss_for_this_checkpoint": 71, "contextualis": 71, "95": 71, "log_x": 71, "302m": 72, "4096": 72, "708m": 72, "1280": 72, "5120": 72, "1600": 72, "6400": 72, "42m": 72, "2048": 72, "50272": 72, "8192": 72, "2560": 72, "10240": 72, "128": 72, "16384": 72, "20480": 72, "7168": 72, "28672": 72, "9216": 72, "36864": 72, "50400": 72, "6144": 72, "50432": 72, "96": 72, "24576": 72, "2m": 72, "50304": 72, "7m": 72, "805m": 72, "50688": 72, "50278": 72, "736": 72, "2944": 72, "101m": 72, "197m": 72, "1536": 72, "48262": 72, "4m": 72, "0m": 72, "50277": 72, "524k": 72, "50259": 72, "32000": 72, "11008": 72, "13824": 72, "6656": 72, "17920": 72, "22016": 72, "78b": 72, "32016": 72, "128256": 72, "14336": 72, "25m": 72, "28996": 72, "393k": 72, "6m": 72, "131072": 72, "47b": 72, "32768": 72, "250880": 72, "679m": 72, "0b": 72, "49280": 72, "151936": 72, "5504": 72, "152064": 72, "13696": 72, "308m": 72, "2816": 72, "6912": 72, "391m": 72, "896": 72, "4864": 72, "8960": 72, "3584": 72, "18944": 72, "27648": 72, "80b": 72, "29568": 72, "51200": 72, "32064": 72, "256000": 72, "2304": 72, "gelu_pytorch_tanh": 72, "4608": 72, "64000": 72, "39b": 72, "32128": 72, "100000": 72, "formerli": 73, "transfer": 73, "courtesi": 73, "austin": 73, "kozlowski": 73}, "objects": {"transformer_lens": [[10, 0, 0, "-", "ActivationCache"], [11, 0, 0, "-", "FactoredMatrix"], [12, 0, 0, "-", "HookedEncoder"], [13, 0, 0, "-", "HookedEncoderDecoder"], [14, 0, 0, "-", "HookedTransformer"], [15, 0, 0, "-", "HookedTransformerConfig"], [16, 0, 0, "-", "SVDInterpreter"], [35, 0, 0, "-", "evals"], [36, 0, 0, "-", "head_detector"], [37, 0, 0, "-", "hook_points"], [38, 0, 0, "-", "loading_from_pretrained"], [39, 0, 0, "-", "past_key_value_caching"], [40, 0, 0, "-", "patching"], [63, 0, 0, "-", "train"], [69, 0, 0, "-", "utils"]], "transformer_lens.ActivationCache": [[10, 1, 1, "", "ActivationCache"]], "transformer_lens.ActivationCache.ActivationCache": [[10, 2, 1, "", "accumulated_resid"], [10, 2, 1, "", "apply_ln_to_stack"], [10, 2, 1, "", "apply_slice_to_batch_dim"], [10, 2, 1, "", "compute_head_results"], [10, 2, 1, "", "decompose_resid"], [10, 2, 1, "", "get_full_resid_decomposition"], [10, 2, 1, "", "get_neuron_results"], [10, 2, 1, "", "items"], [10, 2, 1, "", "keys"], [10, 2, 1, "", "logit_attrs"], [10, 2, 1, "", "remove_batch_dim"], [10, 2, 1, "", "stack_activation"], [10, 2, 1, "", "stack_head_results"], [10, 2, 1, "", "stack_neuron_results"], [10, 2, 1, "", "to"], [10, 2, 1, "", "toggle_autodiff"], [10, 2, 1, "", "values"]], "transformer_lens.FactoredMatrix": [[11, 1, 1, "", "FactoredMatrix"]], "transformer_lens.FactoredMatrix.FactoredMatrix": [[11, 3, 1, "", "AB"], [11, 3, 1, "", "BA"], [11, 3, 1, "", "S"], [11, 3, 1, "", "T"], [11, 3, 1, "", "U"], [11, 3, 1, "", "Vh"], [11, 2, 1, "", "collapse_l"], [11, 2, 1, "", "collapse_r"], [11, 3, 1, "", "eigenvalues"], [11, 2, 1, "", "get_corner"], [11, 2, 1, "", "make_even"], [11, 3, 1, "", "ndim"], [11, 2, 1, "", "norm"], [11, 3, 1, "", "pair"], [11, 2, 1, "", "svd"], [11, 2, 1, "", "unsqueeze"]], "transformer_lens.HookedEncoder": [[12, 1, 1, "", "HookedEncoder"]], "transformer_lens.HookedEncoder.HookedEncoder": [[12, 3, 1, "", "OV"], [12, 3, 1, "", "QK"], [12, 3, 1, "", "W_E"], [12, 3, 1, "", "W_E_pos"], [12, 3, 1, "", "W_K"], [12, 3, 1, "", "W_O"], [12, 3, 1, "", "W_Q"], [12, 3, 1, "", "W_U"], [12, 3, 1, "", "W_V"], [12, 3, 1, "", "W_in"], [12, 3, 1, "", "W_out"], [12, 3, 1, "", "W_pos"], [12, 2, 1, "", "all_head_labels"], [12, 3, 1, "", "b_K"], [12, 3, 1, "", "b_O"], [12, 3, 1, "", "b_Q"], [12, 3, 1, "", "b_U"], [12, 3, 1, "", "b_V"], [12, 3, 1, "", "b_in"], [12, 3, 1, "", "b_out"], [12, 2, 1, "", "cpu"], [12, 2, 1, "", "cuda"], [12, 2, 1, "", "forward"], [12, 2, 1, "", "from_pretrained"], [12, 2, 1, "", "mps"], [12, 2, 1, "", "run_with_cache"], [12, 2, 1, "", "to"]], "transformer_lens.HookedEncoderDecoder": [[13, 1, 1, "", "HookedEncoderDecoder"]], "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder": [[13, 3, 1, "", "OV"], [13, 3, 1, "", "QK"], [13, 3, 1, "", "W_E"], [13, 3, 1, "", "W_K"], [13, 3, 1, "", "W_O"], [13, 3, 1, "", "W_Q"], [13, 3, 1, "", "W_U"], [13, 3, 1, "", "W_V"], [13, 3, 1, "", "W_in"], [13, 3, 1, "", "W_out"], [13, 3, 1, "", "W_pos"], [13, 2, 1, "", "all_head_labels"], [13, 3, 1, "", "b_K"], [13, 3, 1, "", "b_O"], [13, 3, 1, "", "b_Q"], [13, 3, 1, "", "b_U"], [13, 3, 1, "", "b_V"], [13, 3, 1, "", "b_in"], [13, 3, 1, "", "b_out"], [13, 2, 1, "", "cpu"], [13, 2, 1, "", "cuda"], [13, 2, 1, "", "forward"], [13, 2, 1, "", "from_pretrained"], [13, 2, 1, "", "mps"], [13, 2, 1, "", "run_with_cache"], [13, 2, 1, "", "to"]], "transformer_lens.HookedTransformer": [[14, 1, 1, "", "HookedTransformer"], [14, 1, 1, "", "Output"]], "transformer_lens.HookedTransformer.HookedTransformer": [[14, 3, 1, "", "OV"], [14, 3, 1, "", "QK"], [14, 3, 1, "", "W_E"], [14, 3, 1, "", "W_E_pos"], [14, 3, 1, "", "W_K"], [14, 3, 1, "", "W_O"], [14, 3, 1, "", "W_Q"], [14, 3, 1, "", "W_U"], [14, 3, 1, "", "W_V"], [14, 3, 1, "", "W_gate"], [14, 3, 1, "", "W_in"], [14, 3, 1, "", "W_out"], [14, 3, 1, "", "W_pos"], [14, 2, 1, "", "__init__"], [14, 2, 1, "", "accumulated_bias"], [14, 2, 1, "", "all_composition_scores"], [14, 2, 1, "", "all_head_labels"], [14, 3, 1, "", "b_K"], [14, 3, 1, "", "b_O"], [14, 3, 1, "", "b_Q"], [14, 3, 1, "", "b_U"], [14, 3, 1, "", "b_V"], [14, 3, 1, "", "b_in"], [14, 3, 1, "", "b_out"], [14, 2, 1, "", "center_unembed"], [14, 2, 1, "", "center_writing_weights"], [14, 2, 1, "", "check_hooks_to_add"], [14, 2, 1, "", "cpu"], [14, 2, 1, "", "cuda"], [14, 2, 1, "", "fold_layer_norm"], [14, 2, 1, "", "fold_value_biases"], [14, 2, 1, "", "forward"], [14, 2, 1, "", "from_pretrained"], [14, 2, 1, "", "from_pretrained_no_processing"], [14, 2, 1, "", "generate"], [14, 2, 1, "", "get_token_position"], [14, 2, 1, "", "init_weights"], [14, 2, 1, "", "input_to_embed"], [14, 4, 1, "", "ln_final"], [14, 2, 1, "", "load_and_process_state_dict"], [14, 2, 1, "", "load_sample_training_dataset"], [14, 2, 1, "", "loss_fn"], [14, 2, 1, "", "move_model_modules_to_device"], [14, 2, 1, "", "mps"], [14, 2, 1, "", "process_weights_"], [14, 2, 1, "", "refactor_factored_attn_matrices"], [14, 2, 1, "", "run_with_cache"], [14, 2, 1, "", "sample_datapoint"], [14, 2, 1, "", "set_tokenizer"], [14, 2, 1, "", "set_ungroup_grouped_query_attention"], [14, 2, 1, "", "set_use_attn_in"], [14, 2, 1, "", "set_use_attn_result"], [14, 2, 1, "", "set_use_hook_mlp_in"], [14, 2, 1, "", "set_use_split_qkv_input"], [14, 2, 1, "", "to"], [14, 2, 1, "", "to_single_str_token"], [14, 2, 1, "", "to_single_token"], [14, 2, 1, "", "to_str_tokens"], [14, 2, 1, "", "to_string"], [14, 2, 1, "", "to_tokens"], [14, 2, 1, "", "tokens_to_residual_directions"]], "transformer_lens.HookedTransformer.Output": [[14, 4, 1, "", "logits"], [14, 4, 1, "", "loss"]], "transformer_lens.HookedTransformerConfig": [[15, 1, 1, "", "HookedTransformerConfig"]], "transformer_lens.HookedTransformerConfig.HookedTransformerConfig": [[15, 4, 1, "", "NTK_by_parts_factor"], [15, 4, 1, "", "NTK_by_parts_high_freq_factor"], [15, 4, 1, "", "NTK_by_parts_low_freq_factor"], [15, 4, 1, "", "act_fn"], [15, 4, 1, "", "attention_dir"], [15, 4, 1, "", "attn_only"], [15, 4, 1, "", "attn_scale"], [15, 4, 1, "", "attn_scores_soft_cap"], [15, 4, 1, "", "attn_types"], [15, 4, 1, "", "checkpoint_index"], [15, 4, 1, "", "checkpoint_label_type"], [15, 4, 1, "", "checkpoint_value"], [15, 4, 1, "", "d_head"], [15, 4, 1, "", "d_mlp"], [15, 4, 1, "", "d_model"], [15, 4, 1, "", "d_vocab"], [15, 4, 1, "", "d_vocab_out"], [15, 4, 1, "", "decoder_start_token_id"], [15, 4, 1, "", "default_prepend_bos"], [15, 4, 1, "", "device"], [15, 4, 1, "", "dtype"], [15, 4, 1, "", "eps"], [15, 4, 1, "", "experts_per_token"], [15, 4, 1, "", "final_rms"], [15, 4, 1, "", "from_checkpoint"], [15, 2, 1, "", "from_dict"], [15, 4, 1, "", "gated_mlp"], [15, 4, 1, "", "init_mode"], [15, 4, 1, "", "init_weights"], [15, 4, 1, "", "initializer_range"], [15, 2, 1, "", "is_layer_norm_activation"], [15, 4, 1, "", "load_in_4bit"], [15, 4, 1, "", "model_name"], [15, 4, 1, "", "n_ctx"], [15, 4, 1, "", "n_devices"], [15, 4, 1, "", "n_heads"], [15, 4, 1, "", "n_key_value_heads"], [15, 4, 1, "", "n_layers"], [15, 4, 1, "", "n_params"], [15, 4, 1, "", "normalization_type"], [15, 4, 1, "", "num_experts"], [15, 4, 1, "", "original_architecture"], [15, 4, 1, "", "output_logits_soft_cap"], [15, 4, 1, "", "parallel_attn_mlp"], [15, 4, 1, "", "positional_embedding_type"], [15, 4, 1, "", "post_embedding_ln"], [15, 4, 1, "", "relative_attention_max_distance"], [15, 4, 1, "", "relative_attention_num_buckets"], [15, 4, 1, "", "rotary_adjacent_pairs"], [15, 4, 1, "", "rotary_base"], [15, 4, 1, "", "rotary_dim"], [15, 4, 1, "", "scale_attn_by_inverse_layer_idx"], [15, 4, 1, "", "seed"], [15, 2, 1, "", "set_seed_everywhere"], [15, 4, 1, "", "tie_word_embeddings"], [15, 2, 1, "", "to_dict"], [15, 4, 1, "", "tokenizer_name"], [15, 4, 1, "", "tokenizer_prepends_bos"], [15, 4, 1, "", "trust_remote_code"], [15, 4, 1, "", "ungroup_grouped_query_attention"], [15, 2, 1, "", "unwrap"], [15, 4, 1, "", "use_NTK_by_parts_rope"], [15, 4, 1, "", "use_attn_in"], [15, 4, 1, "", "use_attn_result"], [15, 4, 1, "", "use_attn_scale"], [15, 4, 1, "", "use_hook_mlp_in"], [15, 4, 1, "", "use_hook_tokens"], [15, 4, 1, "", "use_local_attn"], [15, 4, 1, "", "use_normalization_before_and_after"], [15, 4, 1, "", "use_split_qkv_input"], [15, 4, 1, "", "window_size"]], "transformer_lens.SVDInterpreter": [[16, 1, 1, "", "SVDInterpreter"]], "transformer_lens.SVDInterpreter.SVDInterpreter": [[16, 2, 1, "", "get_singular_vectors"]], "transformer_lens.components": [[18, 0, 0, "-", "abstract_attention"], [19, 0, 0, "-", "attention"], [20, 0, 0, "-", "bert_block"], [21, 0, 0, "-", "bert_embed"], [22, 0, 0, "-", "bert_mlm_head"], [23, 0, 0, "-", "embed"], [24, 0, 0, "-", "grouped_query_attention"], [25, 0, 0, "-", "layer_norm"], [26, 0, 0, "-", "layer_norm_pre"], [27, 0, 0, "-", "pos_embed"], [28, 0, 0, "-", "rms_norm"], [29, 0, 0, "-", "rms_norm_pre"], [30, 0, 0, "-", "t5_attention"], [31, 0, 0, "-", "t5_block"], [32, 0, 0, "-", "token_typed_embed"], [33, 0, 0, "-", "transformer_block"], [34, 0, 0, "-", "unembed"]], "transformer_lens.components.abstract_attention": [[18, 1, 1, "", "AbstractAttention"]], "transformer_lens.components.abstract_attention.AbstractAttention": [[18, 3, 1, "", "OV"], [18, 3, 1, "", "QK"], [18, 2, 1, "", "__init__"], [18, 4, 1, "", "alibi"], [18, 2, 1, "", "apply_causal_mask"], [18, 2, 1, "", "apply_rotary"], [18, 2, 1, "", "calculate_attention_scores"], [18, 2, 1, "", "calculate_qkv_matrices"], [18, 2, 1, "", "calculate_sin_cos_rotary"], [18, 2, 1, "", "calculate_z_scores"], [18, 2, 1, "", "create_alibi_bias"], [18, 2, 1, "", "create_alibi_multipliers"], [18, 2, 1, "", "create_alibi_slope"], [18, 2, 1, "", "forward"], [18, 2, 1, "", "rotate_every_two"]], "transformer_lens.components.attention": [[19, 1, 1, "", "Attention"]], "transformer_lens.components.attention.Attention": [[19, 2, 1, "", "__init__"]], "transformer_lens.components.bert_block": [[20, 1, 1, "", "BertBlock"]], "transformer_lens.components.bert_block.BertBlock": [[20, 2, 1, "", "forward"]], "transformer_lens.components.bert_embed": [[21, 1, 1, "", "BertEmbed"]], "transformer_lens.components.bert_embed.BertEmbed": [[21, 2, 1, "", "forward"]], "transformer_lens.components.bert_mlm_head": [[22, 1, 1, "", "BertMLMHead"]], "transformer_lens.components.bert_mlm_head.BertMLMHead": [[22, 2, 1, "", "forward"]], "transformer_lens.components.embed": [[23, 1, 1, "", "Embed"]], "transformer_lens.components.embed.Embed": [[23, 2, 1, "", "forward"]], "transformer_lens.components.grouped_query_attention": [[24, 1, 1, "", "GroupedQueryAttention"]], "transformer_lens.components.grouped_query_attention.GroupedQueryAttention": [[24, 3, 1, "", "W_K"], [24, 3, 1, "", "W_V"], [24, 2, 1, "", "__init__"], [24, 3, 1, "", "b_K"], [24, 3, 1, "", "b_V"], [24, 2, 1, "", "calculate_attention_scores"], [24, 2, 1, "", "calculate_qkv_matrices"], [24, 2, 1, "", "calculate_z_scores"]], "transformer_lens.components.layer_norm": [[25, 1, 1, "", "LayerNorm"]], "transformer_lens.components.layer_norm.LayerNorm": [[25, 2, 1, "", "__init__"], [25, 2, 1, "", "forward"]], "transformer_lens.components.layer_norm_pre": [[26, 1, 1, "", "LayerNormPre"]], "transformer_lens.components.layer_norm_pre.LayerNormPre": [[26, 2, 1, "", "__init__"], [26, 2, 1, "", "forward"]], "transformer_lens.components.pos_embed": [[27, 1, 1, "", "PosEmbed"]], "transformer_lens.components.pos_embed.PosEmbed": [[27, 2, 1, "", "forward"]], "transformer_lens.components.rms_norm": [[28, 1, 1, "", "RMSNorm"]], "transformer_lens.components.rms_norm.RMSNorm": [[28, 2, 1, "", "__init__"], [28, 2, 1, "", "forward"]], "transformer_lens.components.rms_norm_pre": [[29, 1, 1, "", "RMSNormPre"]], "transformer_lens.components.rms_norm_pre.RMSNormPre": [[29, 2, 1, "", "__init__"], [29, 2, 1, "", "forward"]], "transformer_lens.components.t5_attention": [[30, 1, 1, "", "T5Attention"]], "transformer_lens.components.t5_attention.T5Attention": [[30, 2, 1, "", "compute_relative_attention_bias"]], "transformer_lens.components.t5_block": [[31, 1, 1, "", "T5Block"]], "transformer_lens.components.t5_block.T5Block": [[31, 2, 1, "", "forward"]], "transformer_lens.components.token_typed_embed": [[32, 1, 1, "", "TokenTypeEmbed"]], "transformer_lens.components.token_typed_embed.TokenTypeEmbed": [[32, 2, 1, "", "forward"]], "transformer_lens.components.transformer_block": [[33, 1, 1, "", "TransformerBlock"]], "transformer_lens.components.transformer_block.TransformerBlock": [[33, 2, 1, "", "apply_mlp"], [33, 2, 1, "", "forward"], [33, 4, 1, "", "ln1"], [33, 4, 1, "", "ln2"], [33, 4, 1, "", "mlp"]], "transformer_lens.components.unembed": [[34, 1, 1, "", "Unembed"]], "transformer_lens.components.unembed.Unembed": [[34, 2, 1, "", "forward"]], "transformer_lens.evals": [[35, 1, 1, "", "IOIDataset"], [35, 5, 1, "", "evaluate"], [35, 5, 1, "", "evaluate_on_dataset"], [35, 5, 1, "", "induction_loss"], [35, 5, 1, "", "ioi_eval"], [35, 5, 1, "", "make_code_data_loader"], [35, 5, 1, "", "make_owt_data_loader"], [35, 5, 1, "", "make_pile_data_loader"], [35, 5, 1, "", "make_wiki_data_loader"], [35, 5, 1, "", "sanity_check"]], "transformer_lens.evals.IOIDataset": [[35, 2, 1, "", "get_default_names"], [35, 2, 1, "", "get_default_nouns"], [35, 2, 1, "", "get_default_templates"], [35, 2, 1, "", "get_sample"]], "transformer_lens.head_detector": [[36, 5, 1, "", "compute_head_attention_similarity_score"], [36, 5, 1, "", "detect_head"], [36, 5, 1, "", "get_duplicate_token_head_detection_pattern"], [36, 5, 1, "", "get_induction_head_detection_pattern"], [36, 5, 1, "", "get_previous_token_head_detection_pattern"], [36, 5, 1, "", "get_supported_heads"]], "transformer_lens.hook_points": [[37, 4, 1, "", "HookFunction"], [37, 1, 1, "", "HookPoint"], [37, 1, 1, "", "HookedRootModule"], [37, 1, 1, "", "LensHandle"]], "transformer_lens.hook_points.HookPoint": [[37, 2, 1, "", "add_hook"], [37, 2, 1, "", "add_perma_hook"], [37, 2, 1, "", "clear_context"], [37, 2, 1, "", "forward"], [37, 2, 1, "", "layer"], [37, 2, 1, "", "remove_hooks"]], "transformer_lens.hook_points.HookedRootModule": [[37, 2, 1, "", "add_caching_hooks"], [37, 2, 1, "", "add_hook"], [37, 2, 1, "", "add_perma_hook"], [37, 2, 1, "", "cache_all"], [37, 2, 1, "", "cache_some"], [37, 2, 1, "", "check_and_add_hook"], [37, 2, 1, "", "check_hooks_to_add"], [37, 2, 1, "", "clear_contexts"], [37, 2, 1, "", "get_caching_hooks"], [37, 4, 1, "", "hook_dict"], [37, 2, 1, "", "hook_points"], [37, 2, 1, "", "hooks"], [37, 4, 1, "", "mod_dict"], [37, 4, 1, "", "name"], [37, 2, 1, "", "remove_all_hook_fns"], [37, 2, 1, "", "reset_hooks"], [37, 2, 1, "", "run_with_cache"], [37, 2, 1, "", "run_with_hooks"], [37, 2, 1, "", "setup"]], "transformer_lens.hook_points.LensHandle": [[37, 4, 1, "", "context_level"], [37, 4, 1, "", "hook"], [37, 4, 1, "", "is_permanent"]], "transformer_lens.loading_from_pretrained": [[38, 1, 1, "", "Config"], [38, 6, 1, "", "MODEL_ALIASES"], [38, 6, 1, "", "NON_HF_HOSTED_MODEL_NAMES"], [38, 6, 1, "", "OFFICIAL_MODEL_NAMES"], [38, 5, 1, "", "get_checkpoint_labels"], [38, 5, 1, "", "get_num_params_of_pretrained"], [38, 5, 1, "", "get_pretrained_model_config"]], "transformer_lens.loading_from_pretrained.Config": [[38, 4, 1, "", "d_head"], [38, 4, 1, "", "d_mlp"], [38, 4, 1, "", "d_model"], [38, 4, 1, "", "d_vocab"], [38, 4, 1, "", "debug"], [38, 4, 1, "", "init_range"], [38, 4, 1, "", "layer_norm_eps"], [38, 4, 1, "", "n_ctx"], [38, 4, 1, "", "n_heads"], [38, 4, 1, "", "n_layers"]], "transformer_lens.past_key_value_caching": [[39, 1, 1, "", "HookedTransformerKeyValueCache"], [39, 1, 1, "", "HookedTransformerKeyValueCacheEntry"]], "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache": [[39, 2, 1, "", "append_attention_mask"], [39, 4, 1, "", "entries"], [39, 2, 1, "", "freeze"], [39, 4, 1, "", "frozen"], [39, 2, 1, "", "init_cache"], [39, 4, 1, "", "previous_attention_mask"], [39, 2, 1, "", "unfreeze"]], "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry": [[39, 2, 1, "", "append"], [39, 4, 1, "", "frozen"], [39, 2, 1, "", "init_cache_entry"], [39, 4, 1, "", "past_keys"], [39, 4, 1, "", "past_values"]], "transformer_lens.patching": [[40, 5, 1, "", "generic_activation_patch"], [40, 5, 1, "", "get_act_patch_attn_head_all_pos_every"], [40, 5, 1, "", "get_act_patch_attn_head_by_pos_every"], [40, 5, 1, "", "get_act_patch_attn_head_k_all_pos"], [40, 5, 1, "", "get_act_patch_attn_head_k_by_pos"], [40, 5, 1, "", "get_act_patch_attn_head_out_all_pos"], [40, 5, 1, "", "get_act_patch_attn_head_out_by_pos"], [40, 5, 1, "", "get_act_patch_attn_head_pattern_all_pos"], [40, 5, 1, "", "get_act_patch_attn_head_pattern_by_pos"], [40, 5, 1, "", "get_act_patch_attn_head_pattern_dest_src_pos"], [40, 5, 1, "", "get_act_patch_attn_head_q_all_pos"], [40, 5, 1, "", "get_act_patch_attn_head_q_by_pos"], [40, 5, 1, "", "get_act_patch_attn_head_v_all_pos"], [40, 5, 1, "", "get_act_patch_attn_head_v_by_pos"], [40, 5, 1, "", "get_act_patch_attn_out"], [40, 5, 1, "", "get_act_patch_block_every"], [40, 5, 1, "", "get_act_patch_mlp_out"], [40, 5, 1, "", "get_act_patch_resid_mid"], [40, 5, 1, "", "get_act_patch_resid_pre"], [40, 5, 1, "", "layer_head_dest_src_pos_pattern_patch_setter"], [40, 5, 1, "", "layer_head_pattern_patch_setter"], [40, 5, 1, "", "layer_head_pos_pattern_patch_setter"], [40, 5, 1, "", "layer_head_vector_patch_setter"], [40, 5, 1, "", "layer_pos_head_vector_patch_setter"], [40, 5, 1, "", "layer_pos_patch_setter"]], "transformer_lens.pretrained.weight_conversions": [[43, 0, 0, "-", "bert"], [44, 0, 0, "-", "bloom"], [45, 0, 0, "-", "coder"], [46, 0, 0, "-", "gemma"], [47, 0, 0, "-", "gpt2"], [48, 0, 0, "-", "gptj"], [49, 0, 0, "-", "llama"], [50, 0, 0, "-", "mingpt"], [51, 0, 0, "-", "mistral"], [52, 0, 0, "-", "mixtral"], [53, 0, 0, "-", "nanogpt"], [54, 0, 0, "-", "neel_solu_old"], [55, 0, 0, "-", "neo"], [56, 0, 0, "-", "neox"], [57, 0, 0, "-", "opt"], [58, 0, 0, "-", "phi"], [59, 0, 0, "-", "phi3"], [60, 0, 0, "-", "qwen"], [61, 0, 0, "-", "qwen2"], [62, 0, 0, "-", "t5"]], "transformer_lens.pretrained.weight_conversions.bloom": [[44, 5, 1, "", "convert_bloom_weights"]], "transformer_lens.pretrained.weight_conversions.coder": [[45, 5, 1, "", "convert_coder_weights"]], "transformer_lens.pretrained.weight_conversions.mistral": [[51, 5, 1, "", "convert_mistral_weights"]], "transformer_lens.pretrained.weight_conversions.mixtral": [[52, 5, 1, "", "convert_mixtral_weights"]], "transformer_lens.pretrained.weight_conversions.phi": [[58, 5, 1, "", "convert_phi_weights"]], "transformer_lens.pretrained.weight_conversions.phi3": [[59, 5, 1, "", "convert_phi3_weights"]], "transformer_lens.pretrained.weight_conversions.qwen": [[60, 5, 1, "", "convert_qwen_weights"]], "transformer_lens.pretrained.weight_conversions.qwen2": [[61, 5, 1, "", "convert_qwen2_weights"]], "transformer_lens.pretrained.weight_conversions.t5": [[62, 5, 1, "", "convert_t5_weights"]], "transformer_lens.train": [[63, 1, 1, "", "HookedTransformerTrainConfig"], [63, 5, 1, "", "train"]], "transformer_lens.train.HookedTransformerTrainConfig": [[63, 4, 1, "", "batch_size"], [63, 4, 1, "", "device"], [63, 4, 1, "", "lr"], [63, 4, 1, "", "max_grad_norm"], [63, 4, 1, "", "max_steps"], [63, 4, 1, "", "momentum"], [63, 4, 1, "", "num_epochs"], [63, 4, 1, "", "optimizer_name"], [63, 4, 1, "", "print_every"], [63, 4, 1, "", "save_dir"], [63, 4, 1, "", "save_every"], [63, 4, 1, "", "seed"], [63, 4, 1, "", "wandb"], [63, 4, 1, "", "wandb_project_name"], [63, 4, 1, "", "warmup_steps"], [63, 4, 1, "", "weight_decay"]], "transformer_lens.utilities": [[65, 0, 0, "-", "activation_functions"], [66, 0, 0, "-", "addmm"], [67, 0, 0, "-", "attention"], [68, 0, 0, "-", "devices"]], "transformer_lens.utilities.addmm": [[66, 5, 1, "", "batch_addmm"], [66, 5, 1, "", "vanilla_addmm"]], "transformer_lens.utilities.attention": [[67, 5, 1, "", "complex_attn_linear"], [67, 5, 1, "", "simple_attn_linear"]], "transformer_lens.utilities.devices": [[68, 5, 1, "", "get_device_for_block_index"], [68, 5, 1, "", "move_to_and_update_config"]], "transformer_lens.utils": [[69, 1, 1, "", "LocallyOverridenDefaults"], [69, 1, 1, "", "Slice"], [69, 6, 1, "", "SliceInput"], [69, 5, 1, "", "calc_fan_in_and_fan_out"], [69, 5, 1, "", "composition_scores"], [69, 5, 1, "", "download_file_from_hf"], [69, 5, 1, "", "gelu_fast"], [69, 5, 1, "", "gelu_new"], [69, 5, 1, "", "get_act_name"], [69, 5, 1, "", "get_attention_mask"], [69, 5, 1, "", "get_corner"], [69, 5, 1, "", "get_cumsum_along_dim"], [69, 5, 1, "", "get_dataset"], [69, 5, 1, "", "get_device"], [69, 5, 1, "", "get_input_with_manually_prepended_bos"], [69, 5, 1, "", "get_nested_attr"], [69, 5, 1, "", "get_offset_position_ids"], [69, 5, 1, "", "get_tokenizer_with_bos"], [69, 5, 1, "", "get_tokens_with_bos_removed"], [69, 5, 1, "", "init_kaiming_normal_"], [69, 5, 1, "", "init_kaiming_uniform_"], [69, 5, 1, "", "init_xavier_normal_"], [69, 5, 1, "", "init_xavier_uniform_"], [69, 5, 1, "", "is_lower_triangular"], [69, 5, 1, "", "is_square"], [69, 5, 1, "", "keep_single_column"], [69, 5, 1, "", "lm_accuracy"], [69, 5, 1, "", "lm_cross_entropy_loss"], [69, 5, 1, "", "override_or_use_default_value"], [69, 5, 1, "", "print_gpu_mem"], [69, 5, 1, "", "remove_batch_dim"], [69, 5, 1, "", "repeat_along_head_dimension"], [69, 5, 1, "", "sample_logits"], [69, 5, 1, "", "set_nested_attr"], [69, 5, 1, "", "solu"], [69, 5, 1, "", "test_prompt"], [69, 5, 1, "", "to_numpy"], [69, 5, 1, "", "tokenize_and_concatenate"], [69, 5, 1, "", "transpose"]], "transformer_lens.utils.LocallyOverridenDefaults": [[69, 2, 1, "", "__init__"]], "transformer_lens.utils.Slice": [[69, 2, 1, "", "__init__"], [69, 2, 1, "", "apply"], [69, 2, 1, "", "indices"], [69, 4, 1, "", "slice"], [69, 2, 1, "", "unwrap"]]}, "objtypes": {"0": "py:module", "1": "py:class", "2": "py:method", "3": "py:property", "4": "py:attribute", "5": "py:function", "6": "py:data"}, "objnames": {"0": ["py", "module", "Python module"], "1": ["py", "class", "Python class"], "2": ["py", "method", "Python method"], "3": ["py", "property", "Python property"], "4": ["py", "attribute", "Python attribute"], "5": ["py", "function", "Python function"], "6": ["py", "data", "Python data"]}, "titleterms": {"citat": 0, "contribut": 1, "setup": [1, 70, 71], "devcontain": 1, "manual": 1, "test": [1, 5], "run": [1, 71], "format": 1, "document": 1, "docstr": 1, "style": 1, "guid": 1, "section": 1, "order": 1, "support": 1, "sphinx": 1, "properti": [1, 72], "refer": 1, "other": [1, 71], "function": [1, 70], "class": [1, 71], "math": 1, "markup": 1, "galleri": 2, "get": [3, 4], "start": [3, 4, 7], "advic": 3, "read": [3, 70], "code": 3, "instal": 3, "huggingfac": 3, "gate": 3, "access": [3, 71], "mechanist": [4, 73], "interpret": [4, 71, 73], "transformerlen": [5, 73], "2": 5, "0": 5, "first": 5, "an": [5, 71], "introduct": [5, 70, 71], "adopt": 5, "semant": 5, "version": 5, "deprec": 5, "roadmap": 5, "immedi": 5, "within": 5, "next": 5, "month": 5, "mid": 5, "term": 5, "3": 5, "perform": 5, "streamlin": 5, "ad": 5, "new": 5, "model": [5, 71, 72, 73], "long": 5, "year": 5, "integr": 5, "contributor": 5, "dev": 5, "branch": 5, "coverag": 5, "compon": [5, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "refactor": 5, "conclus": 5, "appendix": 5, "special": 6, "case": 6, "mixtur": 6, "expert": 6, "error": 6, "rate": 6, "tutori": 7, "where": 7, "To": 7, "demo": [7, 70, 71], "transform": [8, 71], "len": [8, 70, 71], "api": 8, "content": 8, "transformer_len": [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69], "submodul": [9, 17, 42, 64], "subpackag": [9, 41], "activationcach": 10, "factoredmatrix": 11, "hookedencod": 12, "hookedencoderdecod": 13, "hookedtransform": 14, "hookedtransformerconfig": 15, "svdinterpret": 16, "abstract_attent": 18, "attent": [19, 67, 70], "bert_block": 20, "bert_emb": 21, "bert_mlm_head": 22, "emb": 23, "grouped_query_attent": 24, "layer_norm": 25, "layer_norm_pr": 26, "pos_emb": 27, "rms_norm": 28, "rms_norm_pr": 29, "t5_attent": 30, "t5_block": 31, "token_typed_emb": 32, "transformer_block": 33, "unemb": 34, "eval": 35, "head_detector": 36, "hook_point": 37, "loading_from_pretrain": 38, "past_key_value_cach": 39, "patch": [40, 70, 71], "pretrain": [41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62], "weight_convers": [42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62], "bert": 43, "bloom": 44, "coder": 45, "gemma": 46, "gpt2": 47, "gptj": 48, "llama": 49, "mingpt": 50, "mistral": 51, "mixtral": 52, "nanogpt": 53, "neel_solu_old": 54, "neo": 55, "neox": 56, "opt": 57, "phi": 58, "phi3": 59, "qwen": 60, "qwen2": 61, "t5": 62, "train": [63, 71], "util": [64, 65, 66, 67, 68, 69], "activation_funct": 65, "addmm": 66, "devic": 68, "exploratori": 70, "analysi": 70, "tip": 70, "thi": 70, "environ": 70, "ignor": 70, "import": [70, 71], "pytorch": 70, "plot": 70, "helper": 70, "indirect": [70, 71], "object": [70, 71], "identif": [70, 71], "brainstorm": 70, "what": 70, "": 70, "actual": 70, "go": 70, "On": 70, "option": 70, "direct": 70, "logit": 70, "attribut": 70, "layer": 70, "head": [70, 71], "activ": [70, 71], "residu": 70, "stream": 70, "decompos": 70, "consolid": 70, "understand": 70, "visual": 70, "pattern": 70, "compar": 70, "paper": 70, "bonu": 70, "explor": 70, "anomali": 70, "earli": 70, "ar": 70, "induct": [70, 71], "implic": 70, "backup": 70, "name": [70, 71], "mover": 70, "main": 71, "notebook": 71, "load": 71, "cach": 71, "all": 71, "hook": 71, "interven": 71, "task": 71, "avail": 71, "overview": 71, "open": 71, "sourc": 71, "librari": [71, 73], "some": 71, "friendli": 71, "i": 71, "ve": 71, "includ": 71, "resourc": 71, "architectur": 71, "paramet": 71, "fold": 71, "layernorm": 71, "For": 71, "curiou": 71, "featur": 71, "deal": 71, "token": 71, "gotcha": 71, "prepend_bo": 71, "factor": 71, "matrix": 71, "basic": 71, "exampl": 71, "medium": 71, "eigenvalu": 71, "copi": 71, "score": 71, "gener": [71, 73], "text": 71, "point": 71, "toi": 71, "pre": 71, "checkpoint": 71, "phase": 71, "transit": 71, "tabl": 72, "A": 73, "languag": 73}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 8, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "nbsphinx": 4, "sphinx": 57}, "alltitles": {"Citation": [[0, "citation"]], "Contributing": [[1, "contributing"]], "Setup": [[1, "setup"], [70, "Setup"], [71, "Setup"]], "DevContainer": [[1, "devcontainer"]], "Manual Setup": [[1, "manual-setup"]], "Testing": [[1, "testing"]], "Running the tests": [[1, "running-the-tests"]], "Formatting": [[1, "formatting"]], "Documentation": [[1, "documentation"]], "Docstring Style Guide": [[1, "docstring-style-guide"]], "Sections and Order": [[1, "sections-and-order"]], "Supported Sphinx Properties": [[1, "supported-sphinx-properties"]], "References to Other Functions/Classes": [[1, "references-to-other-functions-classes"]], "Maths": [[1, "maths"]], "Markup": [[1, "markup"]], "Gallery": [[2, "gallery"]], "Getting Started": [[3, "getting-started"]], "Advice for Reading the Code": [[3, "advice-for-reading-the-code"]], "Installation": [[3, "installation"]], "Huggingface Gated Access": [[3, "huggingface-gated-access"]], "Getting Started in Mechanistic Interpretability": [[4, "getting-started-in-mechanistic-interpretability"]], "TransformerLens 2.0": [[5, "transformerlens-2-0"]], "First, an introduction": [[5, "first-an-introduction"]], "Adopting Semantic Versioning": [[5, "adopting-semantic-versioning"]], "Deprecations": [[5, "deprecations"]], "Roadmap": [[5, "roadmap"]], "Immediate - within the next month": [[5, "immediate-within-the-next-month"]], "Mid-term - within the next 3 months": [[5, "mid-term-within-the-next-3-months"]], "Performance": [[5, "performance"]], "Streamlining Adding New Models": [[5, "streamlining-adding-new-models"]], "Long-term - within the next year": [[5, "long-term-within-the-next-year"]], "Model Testing": [[5, "model-testing"]], "Model Integration": [[5, "model-integration"]], "Contributors": [[5, "contributors"]], "New Dev Branches": [[5, "new-dev-branches"]], "Integration Tests": [[5, "integration-tests"]], "Test Coverage": [[5, "test-coverage"]], "Components Refactor": [[5, "components-refactor"]], "Conclusion": [[5, "conclusion"]], "Appendix": [[5, "appendix"]], "Semantic Versioning": [[5, "semantic-versioning"]], "Special Cases": [[6, "special-cases"]], "Mixture of Experts error rates": [[6, "mixture-of-experts-error-rates"]], "Tutorials": [[7, "tutorials"]], "Where To Start": [[7, "where-to-start"]], "Demos": [[7, "demos"]], "Transformer Lens API": [[8, "transformer-lens-api"]], "Contents": [[8, "contents"]], "transformer_lens": [[9, "transformer-lens"]], "Submodules": [[9, "submodules"], [17, "submodules"], [42, "submodules"], [64, "submodules"]], "Subpackages": [[9, "subpackages"], [41, "subpackages"]], "transformer_lens.ActivationCache": [[10, "module-transformer_lens.ActivationCache"]], "transformer_lens.FactoredMatrix": [[11, "module-transformer_lens.FactoredMatrix"]], "transformer_lens.HookedEncoder": [[12, "module-transformer_lens.HookedEncoder"]], "transformer_lens.HookedEncoderDecoder": [[13, "module-transformer_lens.HookedEncoderDecoder"]], "transformer_lens.HookedTransformer": [[14, "module-transformer_lens.HookedTransformer"]], "transformer_lens.HookedTransformerConfig": [[15, "module-transformer_lens.HookedTransformerConfig"]], "transformer_lens.SVDInterpreter": [[16, "module-transformer_lens.SVDInterpreter"]], "transformer_lens.components": [[17, "transformer-lens-components"]], "transformer_lens.components.abstract_attention": [[18, "module-transformer_lens.components.abstract_attention"]], "transformer_lens.components.attention": [[19, "module-transformer_lens.components.attention"]], "transformer_lens.components.bert_block": [[20, "module-transformer_lens.components.bert_block"]], "transformer_lens.components.bert_embed": [[21, "module-transformer_lens.components.bert_embed"]], "transformer_lens.components.bert_mlm_head": [[22, "module-transformer_lens.components.bert_mlm_head"]], "transformer_lens.components.embed": [[23, "module-transformer_lens.components.embed"]], "transformer_lens.components.grouped_query_attention": [[24, "module-transformer_lens.components.grouped_query_attention"]], "transformer_lens.components.layer_norm": [[25, "module-transformer_lens.components.layer_norm"]], "transformer_lens.components.layer_norm_pre": [[26, "module-transformer_lens.components.layer_norm_pre"]], "transformer_lens.components.pos_embed": [[27, "module-transformer_lens.components.pos_embed"]], "transformer_lens.components.rms_norm": [[28, "module-transformer_lens.components.rms_norm"]], "transformer_lens.components.rms_norm_pre": [[29, "module-transformer_lens.components.rms_norm_pre"]], "transformer_lens.components.t5_attention": [[30, "module-transformer_lens.components.t5_attention"]], "transformer_lens.components.t5_block": [[31, "module-transformer_lens.components.t5_block"]], "transformer_lens.components.token_typed_embed": [[32, "module-transformer_lens.components.token_typed_embed"]], "transformer_lens.components.transformer_block": [[33, "module-transformer_lens.components.transformer_block"]], "transformer_lens.components.unembed": [[34, "module-transformer_lens.components.unembed"]], "transformer_lens.evals": [[35, "module-transformer_lens.evals"]], "transformer_lens.head_detector": [[36, "module-transformer_lens.head_detector"]], "transformer_lens.hook_points": [[37, "module-transformer_lens.hook_points"]], "transformer_lens.loading_from_pretrained": [[38, "module-transformer_lens.loading_from_pretrained"]], "transformer_lens.past_key_value_caching": [[39, "module-transformer_lens.past_key_value_caching"]], "transformer_lens.patching": [[40, "module-transformer_lens.patching"]], "transformer_lens.pretrained": [[41, "transformer-lens-pretrained"]], "transformer_lens.pretrained.weight_conversions": [[42, "transformer-lens-pretrained-weight-conversions"]], "transformer_lens.pretrained.weight_conversions.bert": [[43, "module-transformer_lens.pretrained.weight_conversions.bert"]], "transformer_lens.pretrained.weight_conversions.bloom": [[44, "module-transformer_lens.pretrained.weight_conversions.bloom"]], "transformer_lens.pretrained.weight_conversions.coder": [[45, "module-transformer_lens.pretrained.weight_conversions.coder"]], "transformer_lens.pretrained.weight_conversions.gemma": [[46, "module-transformer_lens.pretrained.weight_conversions.gemma"]], "transformer_lens.pretrained.weight_conversions.gpt2": [[47, "module-transformer_lens.pretrained.weight_conversions.gpt2"]], "transformer_lens.pretrained.weight_conversions.gptj": [[48, "module-transformer_lens.pretrained.weight_conversions.gptj"]], "transformer_lens.pretrained.weight_conversions.llama": [[49, "module-transformer_lens.pretrained.weight_conversions.llama"]], "transformer_lens.pretrained.weight_conversions.mingpt": [[50, "module-transformer_lens.pretrained.weight_conversions.mingpt"]], "transformer_lens.pretrained.weight_conversions.mistral": [[51, "module-transformer_lens.pretrained.weight_conversions.mistral"]], "transformer_lens.pretrained.weight_conversions.mixtral": [[52, "module-transformer_lens.pretrained.weight_conversions.mixtral"]], "transformer_lens.pretrained.weight_conversions.nanogpt": [[53, "module-transformer_lens.pretrained.weight_conversions.nanogpt"]], "transformer_lens.pretrained.weight_conversions.neel_solu_old": [[54, "module-transformer_lens.pretrained.weight_conversions.neel_solu_old"]], "transformer_lens.pretrained.weight_conversions.neo": [[55, "module-transformer_lens.pretrained.weight_conversions.neo"]], "transformer_lens.pretrained.weight_conversions.neox": [[56, "module-transformer_lens.pretrained.weight_conversions.neox"]], "transformer_lens.pretrained.weight_conversions.opt": [[57, "module-transformer_lens.pretrained.weight_conversions.opt"]], "transformer_lens.pretrained.weight_conversions.phi": [[58, "module-transformer_lens.pretrained.weight_conversions.phi"]], "transformer_lens.pretrained.weight_conversions.phi3": [[59, "module-transformer_lens.pretrained.weight_conversions.phi3"]], "transformer_lens.pretrained.weight_conversions.qwen": [[60, "module-transformer_lens.pretrained.weight_conversions.qwen"]], "transformer_lens.pretrained.weight_conversions.qwen2": [[61, "module-transformer_lens.pretrained.weight_conversions.qwen2"]], "transformer_lens.pretrained.weight_conversions.t5": [[62, "module-transformer_lens.pretrained.weight_conversions.t5"]], "transformer_lens.train": [[63, "module-transformer_lens.train"]], "transformer_lens.utilities": [[64, "transformer-lens-utilities"]], "transformer_lens.utilities.activation_functions": [[65, "module-transformer_lens.utilities.activation_functions"]], "transformer_lens.utilities.addmm": [[66, "module-transformer_lens.utilities.addmm"]], "transformer_lens.utilities.attention": [[67, "module-transformer_lens.utilities.attention"]], "transformer_lens.utilities.devices": [[68, "module-transformer_lens.utilities.devices"]], "transformer_lens.utils": [[69, "module-transformer_lens.utils"]], "Exploratory Analysis Demo": [[70, "Exploratory-Analysis-Demo"]], "Tips for Reading This": [[70, "Tips-for-Reading-This"]], "Environment Setup (ignore)": [[70, "Environment-Setup-(ignore)"]], "Imports": [[70, "Imports"]], "PyTorch Setup": [[70, "PyTorch-Setup"]], "Plotting Helper Functions (ignore)": [[70, "Plotting-Helper-Functions-(ignore)"]], "Introduction": [[70, "Introduction"], [71, "Introduction"]], "Indirect Object Identification": [[70, "Indirect-Object-Identification"]], "Brainstorm What\u2019s Actually Going On (Optional)": [[70, "Brainstorm-What's-Actually-Going-On-(Optional)"]], "Direct Logit Attribution": [[70, "Direct-Logit-Attribution"]], "Logit Lens": [[70, "Logit-Lens"]], "Layer Attribution": [[70, "Layer-Attribution"]], "Head Attribution": [[70, "Head-Attribution"]], "Attention Analysis": [[70, "Attention-Analysis"]], "Activation Patching": [[70, "Activation-Patching"]], "Residual Stream": [[70, "Residual-Stream"]], "Layers": [[70, "Layers"]], "Heads": [[70, "Heads"]], "Decomposing Heads": [[70, "Decomposing-Heads"]], "Consolidating Understanding": [[70, "Consolidating-Understanding"]], "Visualizing Attention Patterns": [[70, "Visualizing-Attention-Patterns"]], "Comparing to the Paper": [[70, "Comparing-to-the-Paper"]], "Bonus: Exploring Anomalies": [[70, "Bonus:-Exploring-Anomalies"]], "Early Heads are Induction Heads(?!)": [[70, "Early-Heads-are-Induction-Heads(?!)"]], "Implications": [[70, "Implications"]], "Backup Name Mover Heads": [[70, "Backup-Name-Mover-Heads"]], "Transformer Lens Main Demo Notebook": [[71, "Transformer-Lens-Main-Demo-Notebook"]], "Loading and Running Models": [[71, "Loading-and-Running-Models"]], "Caching all Activations": [[71, "Caching-all-Activations"]], "Hooks: Intervening on Activations": [[71, "Hooks:-Intervening-on-Activations"]], "Activation Patching on the Indirect Object Identification Task": [[71, "Activation-Patching-on-the-Indirect-Object-Identification-Task"]], "Hooks: Accessing Activations": [[71, "Hooks:-Accessing-Activations"]], "Available Models": [[71, "Available-Models"]], "An overview of the important open source models in the library": [[71, "An-overview-of-the-important-open-source-models-in-the-library"]], "An overview of some interpretability-friendly models I\u2019ve trained and included": [[71, "An-overview-of-some-interpretability-friendly-models-I've-trained-and-included"]], "Other Resources:": [[71, "Other-Resources:"]], "Transformer architecture": [[71, "Transformer-architecture"]], "Parameter Names": [[71, "Parameter-Names"]], "Activation + Hook Names": [[71, "Activation-+-Hook-Names"]], "Folding LayerNorm (For the Curious)": [[71, "Folding-LayerNorm-(For-the-Curious)"]], "Features": [[71, "Features"]], "Dealing with tokens": [[71, "Dealing-with-tokens"]], "Gotcha: prepend_bos": [[71, "Gotcha:-prepend_bos"]], "Factored Matrix Class": [[71, "Factored-Matrix-Class"]], "Basic Examples": [[71, "Basic-Examples"]], "Medium Example: Eigenvalue Copying Scores": [[71, "Medium-Example:-Eigenvalue-Copying-Scores"]], "Generating Text": [[71, "Generating-Text"]], "Hook Points": [[71, "Hook-Points"]], "Toy Example": [[71, "Toy-Example"]], "Loading Pre-Trained Checkpoints": [[71, "Loading-Pre-Trained-Checkpoints"]], "Example: Induction Head Phase Transition": [[71, "Example:-Induction-Head-Phase-Transition"]], "Model Properties Table": [[72, "model-properties-table"]], "TransformerLens": [[73, "transformerlens"]], "A Library for Mechanistic Interpretability of Generative Language Models": [[73, "a-library-for-mechanistic-interpretability-of-generative-language-models"]]}, "indexentries": {"activationcache (class in transformer_lens.activationcache)": [[10, "transformer_lens.ActivationCache.ActivationCache"]], "accumulated_resid() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.accumulated_resid"]], "apply_ln_to_stack() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.apply_ln_to_stack"]], "apply_slice_to_batch_dim() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.apply_slice_to_batch_dim"]], "compute_head_results() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.compute_head_results"]], "decompose_resid() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.decompose_resid"]], "get_full_resid_decomposition() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.get_full_resid_decomposition"]], "get_neuron_results() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.get_neuron_results"]], "items() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.items"]], "keys() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.keys"]], "logit_attrs() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.logit_attrs"]], "module": [[10, "module-transformer_lens.ActivationCache"], [11, "module-transformer_lens.FactoredMatrix"], [12, "module-transformer_lens.HookedEncoder"], [13, "module-transformer_lens.HookedEncoderDecoder"], [14, "module-transformer_lens.HookedTransformer"], [15, "module-transformer_lens.HookedTransformerConfig"], [16, "module-transformer_lens.SVDInterpreter"], [18, "module-transformer_lens.components.abstract_attention"], [19, "module-transformer_lens.components.attention"], [20, "module-transformer_lens.components.bert_block"], [21, "module-transformer_lens.components.bert_embed"], [22, "module-transformer_lens.components.bert_mlm_head"], [23, "module-transformer_lens.components.embed"], [24, "module-transformer_lens.components.grouped_query_attention"], [25, "module-transformer_lens.components.layer_norm"], [26, "module-transformer_lens.components.layer_norm_pre"], [27, "module-transformer_lens.components.pos_embed"], [28, "module-transformer_lens.components.rms_norm"], [29, "module-transformer_lens.components.rms_norm_pre"], [30, "module-transformer_lens.components.t5_attention"], [31, "module-transformer_lens.components.t5_block"], [32, "module-transformer_lens.components.token_typed_embed"], [33, "module-transformer_lens.components.transformer_block"], [34, "module-transformer_lens.components.unembed"], [35, "module-transformer_lens.evals"], [36, "module-transformer_lens.head_detector"], [37, "module-transformer_lens.hook_points"], [38, "module-transformer_lens.loading_from_pretrained"], [39, "module-transformer_lens.past_key_value_caching"], [40, "module-transformer_lens.patching"], [43, "module-transformer_lens.pretrained.weight_conversions.bert"], [44, "module-transformer_lens.pretrained.weight_conversions.bloom"], [45, "module-transformer_lens.pretrained.weight_conversions.coder"], [46, "module-transformer_lens.pretrained.weight_conversions.gemma"], [47, "module-transformer_lens.pretrained.weight_conversions.gpt2"], [48, "module-transformer_lens.pretrained.weight_conversions.gptj"], [49, "module-transformer_lens.pretrained.weight_conversions.llama"], [50, "module-transformer_lens.pretrained.weight_conversions.mingpt"], [51, "module-transformer_lens.pretrained.weight_conversions.mistral"], [52, "module-transformer_lens.pretrained.weight_conversions.mixtral"], [53, "module-transformer_lens.pretrained.weight_conversions.nanogpt"], [54, "module-transformer_lens.pretrained.weight_conversions.neel_solu_old"], [55, "module-transformer_lens.pretrained.weight_conversions.neo"], [56, "module-transformer_lens.pretrained.weight_conversions.neox"], [57, "module-transformer_lens.pretrained.weight_conversions.opt"], [58, "module-transformer_lens.pretrained.weight_conversions.phi"], [59, "module-transformer_lens.pretrained.weight_conversions.phi3"], [60, "module-transformer_lens.pretrained.weight_conversions.qwen"], [61, "module-transformer_lens.pretrained.weight_conversions.qwen2"], [62, "module-transformer_lens.pretrained.weight_conversions.t5"], [63, "module-transformer_lens.train"], [65, "module-transformer_lens.utilities.activation_functions"], [66, "module-transformer_lens.utilities.addmm"], [67, "module-transformer_lens.utilities.attention"], [68, "module-transformer_lens.utilities.devices"], [69, "module-transformer_lens.utils"]], "remove_batch_dim() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.remove_batch_dim"]], "stack_activation() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.stack_activation"]], "stack_head_results() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.stack_head_results"]], "stack_neuron_results() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.stack_neuron_results"]], "to() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.to"]], "toggle_autodiff() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.toggle_autodiff"]], "transformer_lens.activationcache": [[10, "module-transformer_lens.ActivationCache"]], "values() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.values"]], "ab (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.AB"]], "ba (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.BA"]], "factoredmatrix (class in transformer_lens.factoredmatrix)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix"]], "s (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.S"]], "t (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.T"]], "u (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.U"]], "vh (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.Vh"]], "collapse_l() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.collapse_l"]], "collapse_r() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.collapse_r"]], "eigenvalues (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.eigenvalues"]], "get_corner() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.get_corner"]], "make_even() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.make_even"]], "ndim (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.ndim"]], "norm() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.norm"]], "pair (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.pair"]], "svd() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.svd"]], "transformer_lens.factoredmatrix": [[11, "module-transformer_lens.FactoredMatrix"]], "unsqueeze() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.unsqueeze"]], "hookedencoder (class in transformer_lens.hookedencoder)": [[12, "transformer_lens.HookedEncoder.HookedEncoder"]], "ov (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.OV"]], "qk (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.QK"]], "w_e (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_E"]], "w_e_pos (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_E_pos"]], "w_k (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_K"]], "w_o (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_O"]], "w_q (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_Q"]], "w_u (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_U"]], "w_v (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_V"]], "w_in (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_in"]], "w_out (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_out"]], "w_pos (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_pos"]], "all_head_labels() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.all_head_labels"]], "b_k (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_K"]], "b_o (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_O"]], "b_q (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_Q"]], "b_u (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_U"]], "b_v (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_V"]], "b_in (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_in"]], "b_out (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_out"]], "cpu() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.cpu"]], "cuda() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.cuda"]], "forward() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.forward"]], "from_pretrained() (transformer_lens.hookedencoder.hookedencoder class method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.from_pretrained"]], "mps() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.mps"]], "run_with_cache() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.run_with_cache"]], "to() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.to"]], "transformer_lens.hookedencoder": [[12, "module-transformer_lens.HookedEncoder"]], "hookedencoderdecoder (class in transformer_lens.hookedencoderdecoder)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder"]], "ov (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.OV"]], "qk (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.QK"]], "w_e (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_E"]], "w_k (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_K"]], "w_o (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_O"]], "w_q (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_Q"]], "w_u (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_U"]], "w_v (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_V"]], "w_in (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_in"]], "w_out (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_out"]], "w_pos (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_pos"]], "all_head_labels() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.all_head_labels"]], "b_k (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_K"]], "b_o (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_O"]], "b_q (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_Q"]], "b_u (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_U"]], "b_v (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_V"]], "b_in (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_in"]], "b_out (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_out"]], "cpu() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.cpu"]], "cuda() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.cuda"]], "forward() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.forward"]], "from_pretrained() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder class method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.from_pretrained"]], "mps() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.mps"]], "run_with_cache() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.run_with_cache"]], "to() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.to"]], "transformer_lens.hookedencoderdecoder": [[13, "module-transformer_lens.HookedEncoderDecoder"]], "hookedtransformer (class in transformer_lens.hookedtransformer)": [[14, "transformer_lens.HookedTransformer.HookedTransformer"]], "ov (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.OV"]], "output (class in transformer_lens.hookedtransformer)": [[14, "transformer_lens.HookedTransformer.Output"]], "qk (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.QK"]], "w_e (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_E"]], "w_e_pos (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_E_pos"]], "w_k (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_K"]], "w_o (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_O"]], "w_q (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_Q"]], "w_u (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_U"]], "w_v (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_V"]], "w_gate (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_gate"]], "w_in (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_in"]], "w_out (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_out"]], "w_pos (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_pos"]], "__init__() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.__init__"]], "accumulated_bias() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.accumulated_bias"]], "all_composition_scores() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.all_composition_scores"]], "all_head_labels() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.all_head_labels"]], "b_k (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_K"]], "b_o (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_O"]], "b_q (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_Q"]], "b_u (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_U"]], "b_v (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_V"]], "b_in (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_in"]], "b_out (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_out"]], "center_unembed() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.center_unembed"]], "center_writing_weights() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.center_writing_weights"]], "check_hooks_to_add() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.check_hooks_to_add"]], "cpu() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.cpu"]], "cuda() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.cuda"]], "fold_layer_norm() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.fold_layer_norm"]], "fold_value_biases() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.fold_value_biases"]], "forward() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.forward"]], "from_pretrained() (transformer_lens.hookedtransformer.hookedtransformer class method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.from_pretrained"]], "from_pretrained_no_processing() (transformer_lens.hookedtransformer.hookedtransformer class method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.from_pretrained_no_processing"]], "generate() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.generate"]], "get_token_position() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.get_token_position"]], "init_weights() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.init_weights"]], "input_to_embed() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.input_to_embed"]], "ln_final (transformer_lens.hookedtransformer.hookedtransformer attribute)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.ln_final"]], "load_and_process_state_dict() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.load_and_process_state_dict"]], "load_sample_training_dataset() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.load_sample_training_dataset"]], "logits (transformer_lens.hookedtransformer.output attribute)": [[14, "transformer_lens.HookedTransformer.Output.logits"]], "loss (transformer_lens.hookedtransformer.output attribute)": [[14, "transformer_lens.HookedTransformer.Output.loss"]], "loss_fn() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.loss_fn"]], "move_model_modules_to_device() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.move_model_modules_to_device"]], "mps() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.mps"]], "process_weights_() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.process_weights_"]], "refactor_factored_attn_matrices() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.refactor_factored_attn_matrices"]], "run_with_cache() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.run_with_cache"]], "sample_datapoint() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.sample_datapoint"]], "set_tokenizer() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.set_tokenizer"]], "set_ungroup_grouped_query_attention() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.set_ungroup_grouped_query_attention"]], "set_use_attn_in() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.set_use_attn_in"]], "set_use_attn_result() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.set_use_attn_result"]], "set_use_hook_mlp_in() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.set_use_hook_mlp_in"]], "set_use_split_qkv_input() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.set_use_split_qkv_input"]], "to() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.to"]], "to_single_str_token() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.to_single_str_token"]], "to_single_token() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.to_single_token"]], "to_str_tokens() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.to_str_tokens"]], "to_string() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.to_string"]], "to_tokens() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.to_tokens"]], "tokens_to_residual_directions() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.tokens_to_residual_directions"]], "transformer_lens.hookedtransformer": [[14, "module-transformer_lens.HookedTransformer"]], "hookedtransformerconfig (class in transformer_lens.hookedtransformerconfig)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig"]], "ntk_by_parts_factor (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.NTK_by_parts_factor"]], "ntk_by_parts_high_freq_factor (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.NTK_by_parts_high_freq_factor"]], "ntk_by_parts_low_freq_factor (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.NTK_by_parts_low_freq_factor"]], "act_fn (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.act_fn"]], "attention_dir (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.attention_dir"]], "attn_only (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.attn_only"]], "attn_scale (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.attn_scale"]], "attn_scores_soft_cap (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.attn_scores_soft_cap"]], "attn_types (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.attn_types"]], "checkpoint_index (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.checkpoint_index"]], "checkpoint_label_type (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.checkpoint_label_type"]], "checkpoint_value (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.checkpoint_value"]], "d_head (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.d_head"]], "d_mlp (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.d_mlp"]], "d_model (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.d_model"]], "d_vocab (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.d_vocab"]], "d_vocab_out (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.d_vocab_out"]], "decoder_start_token_id (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.decoder_start_token_id"]], "default_prepend_bos (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.default_prepend_bos"]], "device (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.device"]], "dtype (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.dtype"]], "eps (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.eps"]], "experts_per_token (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.experts_per_token"]], "final_rms (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.final_rms"]], "from_checkpoint (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.from_checkpoint"]], "from_dict() (transformer_lens.hookedtransformerconfig.hookedtransformerconfig class method)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.from_dict"]], "gated_mlp (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.gated_mlp"]], "init_mode (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.init_mode"]], "init_weights (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.init_weights"]], "initializer_range (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.initializer_range"]], "is_layer_norm_activation() (transformer_lens.hookedtransformerconfig.hookedtransformerconfig method)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.is_layer_norm_activation"]], "load_in_4bit (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.load_in_4bit"]], "model_name (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.model_name"]], "n_ctx (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.n_ctx"]], "n_devices (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.n_devices"]], "n_heads (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.n_heads"]], "n_key_value_heads (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.n_key_value_heads"]], "n_layers (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.n_layers"]], "n_params (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.n_params"]], "normalization_type (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.normalization_type"]], "num_experts (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.num_experts"]], "original_architecture (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.original_architecture"]], "output_logits_soft_cap (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.output_logits_soft_cap"]], "parallel_attn_mlp (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.parallel_attn_mlp"]], "positional_embedding_type (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.positional_embedding_type"]], "post_embedding_ln (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.post_embedding_ln"]], "relative_attention_max_distance (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.relative_attention_max_distance"]], "relative_attention_num_buckets (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.relative_attention_num_buckets"]], "rotary_adjacent_pairs (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.rotary_adjacent_pairs"]], "rotary_base (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.rotary_base"]], "rotary_dim (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.rotary_dim"]], "scale_attn_by_inverse_layer_idx (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.scale_attn_by_inverse_layer_idx"]], "seed (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.seed"]], "set_seed_everywhere() (transformer_lens.hookedtransformerconfig.hookedtransformerconfig method)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.set_seed_everywhere"]], "tie_word_embeddings (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.tie_word_embeddings"]], "to_dict() (transformer_lens.hookedtransformerconfig.hookedtransformerconfig method)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.to_dict"]], "tokenizer_name (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.tokenizer_name"]], "tokenizer_prepends_bos (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.tokenizer_prepends_bos"]], "transformer_lens.hookedtransformerconfig": [[15, "module-transformer_lens.HookedTransformerConfig"]], "trust_remote_code (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.trust_remote_code"]], "ungroup_grouped_query_attention (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.ungroup_grouped_query_attention"]], "unwrap() (transformer_lens.hookedtransformerconfig.hookedtransformerconfig class method)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.unwrap"]], "use_ntk_by_parts_rope (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_NTK_by_parts_rope"]], "use_attn_in (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_attn_in"]], "use_attn_result (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_attn_result"]], "use_attn_scale (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_attn_scale"]], "use_hook_mlp_in (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_hook_mlp_in"]], "use_hook_tokens (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_hook_tokens"]], "use_local_attn (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_local_attn"]], "use_normalization_before_and_after (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_normalization_before_and_after"]], "use_split_qkv_input (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_split_qkv_input"]], "window_size (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.window_size"]], "svdinterpreter (class in transformer_lens.svdinterpreter)": [[16, "transformer_lens.SVDInterpreter.SVDInterpreter"]], "get_singular_vectors() (transformer_lens.svdinterpreter.svdinterpreter method)": [[16, "transformer_lens.SVDInterpreter.SVDInterpreter.get_singular_vectors"]], "transformer_lens.svdinterpreter": [[16, "module-transformer_lens.SVDInterpreter"]], "abstractattention (class in transformer_lens.components.abstract_attention)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention"]], "ov (transformer_lens.components.abstract_attention.abstractattention property)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.OV"]], "qk (transformer_lens.components.abstract_attention.abstractattention property)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.QK"]], "__init__() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.__init__"]], "alibi (transformer_lens.components.abstract_attention.abstractattention attribute)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.alibi"]], "apply_causal_mask() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.apply_causal_mask"]], "apply_rotary() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.apply_rotary"]], "calculate_attention_scores() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.calculate_attention_scores"]], "calculate_qkv_matrices() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.calculate_qkv_matrices"]], "calculate_sin_cos_rotary() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.calculate_sin_cos_rotary"]], "calculate_z_scores() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.calculate_z_scores"]], "create_alibi_bias() (transformer_lens.components.abstract_attention.abstractattention static method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.create_alibi_bias"]], "create_alibi_multipliers() (transformer_lens.components.abstract_attention.abstractattention static method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.create_alibi_multipliers"]], "create_alibi_slope() (transformer_lens.components.abstract_attention.abstractattention static method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.create_alibi_slope"]], "forward() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.forward"]], "rotate_every_two() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.rotate_every_two"]], "transformer_lens.components.abstract_attention": [[18, "module-transformer_lens.components.abstract_attention"]], "attention (class in transformer_lens.components.attention)": [[19, "transformer_lens.components.attention.Attention"]], "__init__() (transformer_lens.components.attention.attention method)": [[19, "transformer_lens.components.attention.Attention.__init__"]], "transformer_lens.components.attention": [[19, "module-transformer_lens.components.attention"]], "bertblock (class in transformer_lens.components.bert_block)": [[20, "transformer_lens.components.bert_block.BertBlock"]], "forward() (transformer_lens.components.bert_block.bertblock method)": [[20, "transformer_lens.components.bert_block.BertBlock.forward"]], "transformer_lens.components.bert_block": [[20, "module-transformer_lens.components.bert_block"]], "bertembed (class in transformer_lens.components.bert_embed)": [[21, "transformer_lens.components.bert_embed.BertEmbed"]], "forward() (transformer_lens.components.bert_embed.bertembed method)": [[21, "transformer_lens.components.bert_embed.BertEmbed.forward"]], "transformer_lens.components.bert_embed": [[21, "module-transformer_lens.components.bert_embed"]], "bertmlmhead (class in transformer_lens.components.bert_mlm_head)": [[22, "transformer_lens.components.bert_mlm_head.BertMLMHead"]], "forward() (transformer_lens.components.bert_mlm_head.bertmlmhead method)": [[22, "transformer_lens.components.bert_mlm_head.BertMLMHead.forward"]], "transformer_lens.components.bert_mlm_head": [[22, "module-transformer_lens.components.bert_mlm_head"]], "embed (class in transformer_lens.components.embed)": [[23, "transformer_lens.components.embed.Embed"]], "forward() (transformer_lens.components.embed.embed method)": [[23, "transformer_lens.components.embed.Embed.forward"]], "transformer_lens.components.embed": [[23, "module-transformer_lens.components.embed"]], "groupedqueryattention (class in transformer_lens.components.grouped_query_attention)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention"]], "w_k (transformer_lens.components.grouped_query_attention.groupedqueryattention property)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.W_K"]], "w_v (transformer_lens.components.grouped_query_attention.groupedqueryattention property)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.W_V"]], "__init__() (transformer_lens.components.grouped_query_attention.groupedqueryattention method)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.__init__"]], "b_k (transformer_lens.components.grouped_query_attention.groupedqueryattention property)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.b_K"]], "b_v (transformer_lens.components.grouped_query_attention.groupedqueryattention property)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.b_V"]], "calculate_attention_scores() (transformer_lens.components.grouped_query_attention.groupedqueryattention method)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.calculate_attention_scores"]], "calculate_qkv_matrices() (transformer_lens.components.grouped_query_attention.groupedqueryattention method)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.calculate_qkv_matrices"]], "calculate_z_scores() (transformer_lens.components.grouped_query_attention.groupedqueryattention method)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.calculate_z_scores"]], "transformer_lens.components.grouped_query_attention": [[24, "module-transformer_lens.components.grouped_query_attention"]], "layernorm (class in transformer_lens.components.layer_norm)": [[25, "transformer_lens.components.layer_norm.LayerNorm"]], "__init__() (transformer_lens.components.layer_norm.layernorm method)": [[25, "transformer_lens.components.layer_norm.LayerNorm.__init__"]], "forward() (transformer_lens.components.layer_norm.layernorm method)": [[25, "transformer_lens.components.layer_norm.LayerNorm.forward"]], "transformer_lens.components.layer_norm": [[25, "module-transformer_lens.components.layer_norm"]], "layernormpre (class in transformer_lens.components.layer_norm_pre)": [[26, "transformer_lens.components.layer_norm_pre.LayerNormPre"]], "__init__() (transformer_lens.components.layer_norm_pre.layernormpre method)": [[26, "transformer_lens.components.layer_norm_pre.LayerNormPre.__init__"]], "forward() (transformer_lens.components.layer_norm_pre.layernormpre method)": [[26, "transformer_lens.components.layer_norm_pre.LayerNormPre.forward"]], "transformer_lens.components.layer_norm_pre": [[26, "module-transformer_lens.components.layer_norm_pre"]], "posembed (class in transformer_lens.components.pos_embed)": [[27, "transformer_lens.components.pos_embed.PosEmbed"]], "forward() (transformer_lens.components.pos_embed.posembed method)": [[27, "transformer_lens.components.pos_embed.PosEmbed.forward"]], "transformer_lens.components.pos_embed": [[27, "module-transformer_lens.components.pos_embed"]], "rmsnorm (class in transformer_lens.components.rms_norm)": [[28, "transformer_lens.components.rms_norm.RMSNorm"]], "__init__() (transformer_lens.components.rms_norm.rmsnorm method)": [[28, "transformer_lens.components.rms_norm.RMSNorm.__init__"]], "forward() (transformer_lens.components.rms_norm.rmsnorm method)": [[28, "transformer_lens.components.rms_norm.RMSNorm.forward"]], "transformer_lens.components.rms_norm": [[28, "module-transformer_lens.components.rms_norm"]], "rmsnormpre (class in transformer_lens.components.rms_norm_pre)": [[29, "transformer_lens.components.rms_norm_pre.RMSNormPre"]], "__init__() (transformer_lens.components.rms_norm_pre.rmsnormpre method)": [[29, "transformer_lens.components.rms_norm_pre.RMSNormPre.__init__"]], "forward() (transformer_lens.components.rms_norm_pre.rmsnormpre method)": [[29, "transformer_lens.components.rms_norm_pre.RMSNormPre.forward"]], "transformer_lens.components.rms_norm_pre": [[29, "module-transformer_lens.components.rms_norm_pre"]], "t5attention (class in transformer_lens.components.t5_attention)": [[30, "transformer_lens.components.t5_attention.T5Attention"]], "compute_relative_attention_bias() (transformer_lens.components.t5_attention.t5attention method)": [[30, "transformer_lens.components.t5_attention.T5Attention.compute_relative_attention_bias"]], "transformer_lens.components.t5_attention": [[30, "module-transformer_lens.components.t5_attention"]], "t5block (class in transformer_lens.components.t5_block)": [[31, "transformer_lens.components.t5_block.T5Block"]], "forward() (transformer_lens.components.t5_block.t5block method)": [[31, "transformer_lens.components.t5_block.T5Block.forward"]], "transformer_lens.components.t5_block": [[31, "module-transformer_lens.components.t5_block"]], "tokentypeembed (class in transformer_lens.components.token_typed_embed)": [[32, "transformer_lens.components.token_typed_embed.TokenTypeEmbed"]], "forward() (transformer_lens.components.token_typed_embed.tokentypeembed method)": [[32, "transformer_lens.components.token_typed_embed.TokenTypeEmbed.forward"]], "transformer_lens.components.token_typed_embed": [[32, "module-transformer_lens.components.token_typed_embed"]], "transformerblock (class in transformer_lens.components.transformer_block)": [[33, "transformer_lens.components.transformer_block.TransformerBlock"]], "apply_mlp() (transformer_lens.components.transformer_block.transformerblock method)": [[33, "transformer_lens.components.transformer_block.TransformerBlock.apply_mlp"]], "forward() (transformer_lens.components.transformer_block.transformerblock method)": [[33, "transformer_lens.components.transformer_block.TransformerBlock.forward"]], "ln1 (transformer_lens.components.transformer_block.transformerblock attribute)": [[33, "transformer_lens.components.transformer_block.TransformerBlock.ln1"]], "ln2 (transformer_lens.components.transformer_block.transformerblock attribute)": [[33, "transformer_lens.components.transformer_block.TransformerBlock.ln2"]], "mlp (transformer_lens.components.transformer_block.transformerblock attribute)": [[33, "transformer_lens.components.transformer_block.TransformerBlock.mlp"]], "transformer_lens.components.transformer_block": [[33, "module-transformer_lens.components.transformer_block"]], "unembed (class in transformer_lens.components.unembed)": [[34, "transformer_lens.components.unembed.Unembed"]], "forward() (transformer_lens.components.unembed.unembed method)": [[34, "transformer_lens.components.unembed.Unembed.forward"]], "transformer_lens.components.unembed": [[34, "module-transformer_lens.components.unembed"]], "ioidataset (class in transformer_lens.evals)": [[35, "transformer_lens.evals.IOIDataset"]], "evaluate() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.evaluate"]], "evaluate_on_dataset() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.evaluate_on_dataset"]], "get_default_names() (transformer_lens.evals.ioidataset static method)": [[35, "transformer_lens.evals.IOIDataset.get_default_names"]], "get_default_nouns() (transformer_lens.evals.ioidataset static method)": [[35, "transformer_lens.evals.IOIDataset.get_default_nouns"]], "get_default_templates() (transformer_lens.evals.ioidataset static method)": [[35, "transformer_lens.evals.IOIDataset.get_default_templates"]], "get_sample() (transformer_lens.evals.ioidataset method)": [[35, "transformer_lens.evals.IOIDataset.get_sample"]], "induction_loss() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.induction_loss"]], "ioi_eval() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.ioi_eval"]], "make_code_data_loader() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.make_code_data_loader"]], "make_owt_data_loader() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.make_owt_data_loader"]], "make_pile_data_loader() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.make_pile_data_loader"]], "make_wiki_data_loader() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.make_wiki_data_loader"]], "sanity_check() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.sanity_check"]], "transformer_lens.evals": [[35, "module-transformer_lens.evals"]], "compute_head_attention_similarity_score() (in module transformer_lens.head_detector)": [[36, "transformer_lens.head_detector.compute_head_attention_similarity_score"]], "detect_head() (in module transformer_lens.head_detector)": [[36, "transformer_lens.head_detector.detect_head"]], "get_duplicate_token_head_detection_pattern() (in module transformer_lens.head_detector)": [[36, "transformer_lens.head_detector.get_duplicate_token_head_detection_pattern"]], "get_induction_head_detection_pattern() (in module transformer_lens.head_detector)": [[36, "transformer_lens.head_detector.get_induction_head_detection_pattern"]], "get_previous_token_head_detection_pattern() (in module transformer_lens.head_detector)": [[36, "transformer_lens.head_detector.get_previous_token_head_detection_pattern"]], "get_supported_heads() (in module transformer_lens.head_detector)": [[36, "transformer_lens.head_detector.get_supported_heads"]], "transformer_lens.head_detector": [[36, "module-transformer_lens.head_detector"]], "hookfunction (in module transformer_lens.hook_points)": [[37, "transformer_lens.hook_points.HookFunction"]], "hookpoint (class in transformer_lens.hook_points)": [[37, "transformer_lens.hook_points.HookPoint"]], "hookedrootmodule (class in transformer_lens.hook_points)": [[37, "transformer_lens.hook_points.HookedRootModule"]], "lenshandle (class in transformer_lens.hook_points)": [[37, "transformer_lens.hook_points.LensHandle"]], "add_caching_hooks() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.add_caching_hooks"]], "add_hook() (transformer_lens.hook_points.hookpoint method)": [[37, "transformer_lens.hook_points.HookPoint.add_hook"]], "add_hook() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.add_hook"]], "add_perma_hook() (transformer_lens.hook_points.hookpoint method)": [[37, "transformer_lens.hook_points.HookPoint.add_perma_hook"]], "add_perma_hook() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.add_perma_hook"]], "cache_all() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.cache_all"]], "cache_some() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.cache_some"]], "check_and_add_hook() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.check_and_add_hook"]], "check_hooks_to_add() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.check_hooks_to_add"]], "clear_context() (transformer_lens.hook_points.hookpoint method)": [[37, "transformer_lens.hook_points.HookPoint.clear_context"]], "clear_contexts() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.clear_contexts"]], "context_level (transformer_lens.hook_points.lenshandle attribute)": [[37, "transformer_lens.hook_points.LensHandle.context_level"]], "forward() (transformer_lens.hook_points.hookpoint method)": [[37, "transformer_lens.hook_points.HookPoint.forward"]], "get_caching_hooks() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.get_caching_hooks"]], "hook (transformer_lens.hook_points.lenshandle attribute)": [[37, "transformer_lens.hook_points.LensHandle.hook"]], "hook_dict (transformer_lens.hook_points.hookedrootmodule attribute)": [[37, "transformer_lens.hook_points.HookedRootModule.hook_dict"]], "hook_points() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.hook_points"]], "hooks() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.hooks"]], "is_permanent (transformer_lens.hook_points.lenshandle attribute)": [[37, "transformer_lens.hook_points.LensHandle.is_permanent"]], "layer() (transformer_lens.hook_points.hookpoint method)": [[37, "transformer_lens.hook_points.HookPoint.layer"]], "mod_dict (transformer_lens.hook_points.hookedrootmodule attribute)": [[37, "transformer_lens.hook_points.HookedRootModule.mod_dict"]], "name (transformer_lens.hook_points.hookedrootmodule attribute)": [[37, "transformer_lens.hook_points.HookedRootModule.name"]], "remove_all_hook_fns() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.remove_all_hook_fns"]], "remove_hooks() (transformer_lens.hook_points.hookpoint method)": [[37, "transformer_lens.hook_points.HookPoint.remove_hooks"]], "reset_hooks() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.reset_hooks"]], "run_with_cache() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.run_with_cache"]], "run_with_hooks() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.run_with_hooks"]], "setup() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.setup"]], "transformer_lens.hook_points": [[37, "module-transformer_lens.hook_points"]], "config (class in transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.Config"]], "model_aliases (in module transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.MODEL_ALIASES"]], "non_hf_hosted_model_names (in module transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.NON_HF_HOSTED_MODEL_NAMES"]], "official_model_names (in module transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.OFFICIAL_MODEL_NAMES"]], "d_head (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.d_head"]], "d_mlp (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.d_mlp"]], "d_model (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.d_model"]], "d_vocab (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.d_vocab"]], "debug (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.debug"]], "get_checkpoint_labels() (in module transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.get_checkpoint_labels"]], "get_num_params_of_pretrained() (in module transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.get_num_params_of_pretrained"]], "get_pretrained_model_config() (in module transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.get_pretrained_model_config"]], "init_range (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.init_range"]], "layer_norm_eps (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.layer_norm_eps"]], "n_ctx (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.n_ctx"]], "n_heads (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.n_heads"]], "n_layers (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.n_layers"]], "transformer_lens.loading_from_pretrained": [[38, "module-transformer_lens.loading_from_pretrained"]], "hookedtransformerkeyvaluecache (class in transformer_lens.past_key_value_caching)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache"]], "hookedtransformerkeyvaluecacheentry (class in transformer_lens.past_key_value_caching)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry"]], "append() (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecacheentry method)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry.append"]], "append_attention_mask() (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache method)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.append_attention_mask"]], "entries (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache attribute)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.entries"]], "freeze() (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache method)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.freeze"]], "frozen (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache attribute)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.frozen"]], "frozen (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecacheentry attribute)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry.frozen"]], "init_cache() (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache class method)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.init_cache"]], "init_cache_entry() (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecacheentry class method)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry.init_cache_entry"]], "past_keys (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecacheentry attribute)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry.past_keys"]], "past_values (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecacheentry attribute)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry.past_values"]], "previous_attention_mask (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache attribute)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.previous_attention_mask"]], "transformer_lens.past_key_value_caching": [[39, "module-transformer_lens.past_key_value_caching"]], "unfreeze() (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache method)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.unfreeze"]], "generic_activation_patch() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.generic_activation_patch"]], "get_act_patch_attn_head_all_pos_every() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_all_pos_every"]], "get_act_patch_attn_head_by_pos_every() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_by_pos_every"]], "get_act_patch_attn_head_k_all_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_k_all_pos"]], "get_act_patch_attn_head_k_by_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_k_by_pos"]], "get_act_patch_attn_head_out_all_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_out_all_pos"]], "get_act_patch_attn_head_out_by_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_out_by_pos"]], "get_act_patch_attn_head_pattern_all_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_pattern_all_pos"]], "get_act_patch_attn_head_pattern_by_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_pattern_by_pos"]], "get_act_patch_attn_head_pattern_dest_src_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_pattern_dest_src_pos"]], "get_act_patch_attn_head_q_all_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_q_all_pos"]], "get_act_patch_attn_head_q_by_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_q_by_pos"]], "get_act_patch_attn_head_v_all_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_v_all_pos"]], "get_act_patch_attn_head_v_by_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_v_by_pos"]], "get_act_patch_attn_out() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_out"]], "get_act_patch_block_every() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_block_every"]], "get_act_patch_mlp_out() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_mlp_out"]], "get_act_patch_resid_mid() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_resid_mid"]], "get_act_patch_resid_pre() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_resid_pre"]], "layer_head_dest_src_pos_pattern_patch_setter() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.layer_head_dest_src_pos_pattern_patch_setter"]], "layer_head_pattern_patch_setter() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.layer_head_pattern_patch_setter"]], "layer_head_pos_pattern_patch_setter() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.layer_head_pos_pattern_patch_setter"]], "layer_head_vector_patch_setter() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.layer_head_vector_patch_setter"]], "layer_pos_head_vector_patch_setter() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.layer_pos_head_vector_patch_setter"]], "layer_pos_patch_setter() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.layer_pos_patch_setter"]], "transformer_lens.patching": [[40, "module-transformer_lens.patching"]], "transformer_lens.pretrained.weight_conversions.bert": [[43, "module-transformer_lens.pretrained.weight_conversions.bert"]], "convert_bloom_weights() (in module transformer_lens.pretrained.weight_conversions.bloom)": [[44, "transformer_lens.pretrained.weight_conversions.bloom.convert_bloom_weights"]], "transformer_lens.pretrained.weight_conversions.bloom": [[44, "module-transformer_lens.pretrained.weight_conversions.bloom"]], "convert_coder_weights() (in module transformer_lens.pretrained.weight_conversions.coder)": [[45, "transformer_lens.pretrained.weight_conversions.coder.convert_coder_weights"]], "transformer_lens.pretrained.weight_conversions.coder": [[45, "module-transformer_lens.pretrained.weight_conversions.coder"]], "transformer_lens.pretrained.weight_conversions.gemma": [[46, "module-transformer_lens.pretrained.weight_conversions.gemma"]], "transformer_lens.pretrained.weight_conversions.gpt2": [[47, "module-transformer_lens.pretrained.weight_conversions.gpt2"]], "transformer_lens.pretrained.weight_conversions.gptj": [[48, "module-transformer_lens.pretrained.weight_conversions.gptj"]], "transformer_lens.pretrained.weight_conversions.llama": [[49, "module-transformer_lens.pretrained.weight_conversions.llama"]], "transformer_lens.pretrained.weight_conversions.mingpt": [[50, "module-transformer_lens.pretrained.weight_conversions.mingpt"]], "convert_mistral_weights() (in module transformer_lens.pretrained.weight_conversions.mistral)": [[51, "transformer_lens.pretrained.weight_conversions.mistral.convert_mistral_weights"]], "transformer_lens.pretrained.weight_conversions.mistral": [[51, "module-transformer_lens.pretrained.weight_conversions.mistral"]], "convert_mixtral_weights() (in module transformer_lens.pretrained.weight_conversions.mixtral)": [[52, "transformer_lens.pretrained.weight_conversions.mixtral.convert_mixtral_weights"]], "transformer_lens.pretrained.weight_conversions.mixtral": [[52, "module-transformer_lens.pretrained.weight_conversions.mixtral"]], "transformer_lens.pretrained.weight_conversions.nanogpt": [[53, "module-transformer_lens.pretrained.weight_conversions.nanogpt"]], "transformer_lens.pretrained.weight_conversions.neel_solu_old": [[54, "module-transformer_lens.pretrained.weight_conversions.neel_solu_old"]], "transformer_lens.pretrained.weight_conversions.neo": [[55, "module-transformer_lens.pretrained.weight_conversions.neo"]], "transformer_lens.pretrained.weight_conversions.neox": [[56, "module-transformer_lens.pretrained.weight_conversions.neox"]], "transformer_lens.pretrained.weight_conversions.opt": [[57, "module-transformer_lens.pretrained.weight_conversions.opt"]], "convert_phi_weights() (in module transformer_lens.pretrained.weight_conversions.phi)": [[58, "transformer_lens.pretrained.weight_conversions.phi.convert_phi_weights"]], "transformer_lens.pretrained.weight_conversions.phi": [[58, "module-transformer_lens.pretrained.weight_conversions.phi"]], "convert_phi3_weights() (in module transformer_lens.pretrained.weight_conversions.phi3)": [[59, "transformer_lens.pretrained.weight_conversions.phi3.convert_phi3_weights"]], "transformer_lens.pretrained.weight_conversions.phi3": [[59, "module-transformer_lens.pretrained.weight_conversions.phi3"]], "convert_qwen_weights() (in module transformer_lens.pretrained.weight_conversions.qwen)": [[60, "transformer_lens.pretrained.weight_conversions.qwen.convert_qwen_weights"]], "transformer_lens.pretrained.weight_conversions.qwen": [[60, "module-transformer_lens.pretrained.weight_conversions.qwen"]], "convert_qwen2_weights() (in module transformer_lens.pretrained.weight_conversions.qwen2)": [[61, "transformer_lens.pretrained.weight_conversions.qwen2.convert_qwen2_weights"]], "transformer_lens.pretrained.weight_conversions.qwen2": [[61, "module-transformer_lens.pretrained.weight_conversions.qwen2"]], "convert_t5_weights() (in module transformer_lens.pretrained.weight_conversions.t5)": [[62, "transformer_lens.pretrained.weight_conversions.t5.convert_t5_weights"]], "transformer_lens.pretrained.weight_conversions.t5": [[62, "module-transformer_lens.pretrained.weight_conversions.t5"]], "hookedtransformertrainconfig (class in transformer_lens.train)": [[63, "transformer_lens.train.HookedTransformerTrainConfig"]], "batch_size (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.batch_size"]], "device (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.device"]], "lr (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.lr"]], "max_grad_norm (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.max_grad_norm"]], "max_steps (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.max_steps"]], "momentum (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.momentum"]], "num_epochs (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.num_epochs"]], "optimizer_name (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.optimizer_name"]], "print_every (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.print_every"]], "save_dir (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.save_dir"]], "save_every (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.save_every"]], "seed (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.seed"]], "train() (in module transformer_lens.train)": [[63, "transformer_lens.train.train"]], "transformer_lens.train": [[63, "module-transformer_lens.train"]], "wandb (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.wandb"]], "wandb_project_name (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.wandb_project_name"]], "warmup_steps (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.warmup_steps"]], "weight_decay (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.weight_decay"]], "transformer_lens.utilities.activation_functions": [[65, "module-transformer_lens.utilities.activation_functions"]], "batch_addmm() (in module transformer_lens.utilities.addmm)": [[66, "transformer_lens.utilities.addmm.batch_addmm"]], "transformer_lens.utilities.addmm": [[66, "module-transformer_lens.utilities.addmm"]], "vanilla_addmm() (in module transformer_lens.utilities.addmm)": [[66, "transformer_lens.utilities.addmm.vanilla_addmm"]], "complex_attn_linear() (in module transformer_lens.utilities.attention)": [[67, "transformer_lens.utilities.attention.complex_attn_linear"]], "simple_attn_linear() (in module transformer_lens.utilities.attention)": [[67, "transformer_lens.utilities.attention.simple_attn_linear"]], "transformer_lens.utilities.attention": [[67, "module-transformer_lens.utilities.attention"]], "get_device_for_block_index() (in module transformer_lens.utilities.devices)": [[68, "transformer_lens.utilities.devices.get_device_for_block_index"]], "move_to_and_update_config() (in module transformer_lens.utilities.devices)": [[68, "transformer_lens.utilities.devices.move_to_and_update_config"]], "transformer_lens.utilities.devices": [[68, "module-transformer_lens.utilities.devices"]], "locallyoverridendefaults (class in transformer_lens.utils)": [[69, "transformer_lens.utils.LocallyOverridenDefaults"]], "slice (class in transformer_lens.utils)": [[69, "transformer_lens.utils.Slice"]], "sliceinput (in module transformer_lens.utils)": [[69, "transformer_lens.utils.SliceInput"]], "__init__() (transformer_lens.utils.locallyoverridendefaults method)": [[69, "transformer_lens.utils.LocallyOverridenDefaults.__init__"]], "__init__() (transformer_lens.utils.slice method)": [[69, "transformer_lens.utils.Slice.__init__"]], "apply() (transformer_lens.utils.slice method)": [[69, "transformer_lens.utils.Slice.apply"]], "calc_fan_in_and_fan_out() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.calc_fan_in_and_fan_out"]], "composition_scores() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.composition_scores"]], "download_file_from_hf() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.download_file_from_hf"]], "gelu_fast() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.gelu_fast"]], "gelu_new() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.gelu_new"]], "get_act_name() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_act_name"]], "get_attention_mask() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_attention_mask"]], "get_corner() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_corner"]], "get_cumsum_along_dim() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_cumsum_along_dim"]], "get_dataset() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_dataset"]], "get_device() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_device"]], "get_input_with_manually_prepended_bos() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_input_with_manually_prepended_bos"]], "get_nested_attr() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_nested_attr"]], "get_offset_position_ids() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_offset_position_ids"]], "get_tokenizer_with_bos() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_tokenizer_with_bos"]], "get_tokens_with_bos_removed() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_tokens_with_bos_removed"]], "indices() (transformer_lens.utils.slice method)": [[69, "transformer_lens.utils.Slice.indices"]], "init_kaiming_normal_() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.init_kaiming_normal_"]], "init_kaiming_uniform_() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.init_kaiming_uniform_"]], "init_xavier_normal_() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.init_xavier_normal_"]], "init_xavier_uniform_() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.init_xavier_uniform_"]], "is_lower_triangular() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.is_lower_triangular"]], "is_square() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.is_square"]], "keep_single_column() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.keep_single_column"]], "lm_accuracy() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.lm_accuracy"]], "lm_cross_entropy_loss() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.lm_cross_entropy_loss"]], "override_or_use_default_value() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.override_or_use_default_value"]], "print_gpu_mem() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.print_gpu_mem"]], "remove_batch_dim() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.remove_batch_dim"]], "repeat_along_head_dimension() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.repeat_along_head_dimension"]], "sample_logits() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.sample_logits"]], "set_nested_attr() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.set_nested_attr"]], "slice (transformer_lens.utils.slice attribute)": [[69, "transformer_lens.utils.Slice.slice"]], "solu() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.solu"]], "test_prompt() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.test_prompt"]], "to_numpy() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.to_numpy"]], "tokenize_and_concatenate() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.tokenize_and_concatenate"]], "transformer_lens.utils": [[69, "module-transformer_lens.utils"]], "transpose() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.transpose"]], "unwrap() (transformer_lens.utils.slice class method)": [[69, "transformer_lens.utils.Slice.unwrap"]]}})
\ No newline at end of file