Deploying to gh-pages from @ f103deb 🚀

TransformerLensOrg · Dec 31, 2024 · 201edaa · 201edaa
1 parent c67f384
commit 201edaa
Show file tree

Hide file tree

Showing 78 changed files with 4,893 additions and 4,878 deletions.
diff --git a/_sources/generated/model_properties_table.md.txt b/_sources/generated/model_properties_table.md.txt
@@ -109,14 +109,15 @@
 | meta-llama/Meta-Llama-3-8B-Instruct  | 7.8B       |         32 |      4096 |        32 | silu              |    8192 |    128256 |      128 |   14336 | 8                   |
 | meta-llama/Meta-Llama-3-70B          | 78B        |         80 |      8192 |        64 | silu              |    8192 |    128256 |      128 |   28672 | 8                   |
 | meta-llama/Meta-Llama-3-70B-Instruct | 78B        |         80 |      8192 |        64 | silu              |    8192 |    128256 |      128 |   28672 | 8                   |
-| meta-llama/Llama-3.2-1B              | 1.1B       |         16 |      2048 |        32 | silu              |    2048 |    128256 |       64 |    8192 | 8                   |
-| meta-llama/Llama-3.2-3B              | 3.2B       |         28 |      3072 |        24 | silu              |    2048 |    128256 |      128 |    8192 | 8                   |
-| meta-llama/Llama-3.2-1B-Instruct     | 1.1B       |         16 |      2048 |        32 | silu              |    2048 |    128256 |       64 |    8192 | 8                   |
-| meta-llama/Llama-3.2-3B-Instruct     | 3.2B       |         28 |      3072 |        24 | silu              |    2048 |    128256 |      128 |    8192 | 8                   |
 | meta-llama/Llama-3.1-70B             | 78B        |         80 |      8192 |        64 | silu              |    2048 |    128256 |      128 |   28672 | 8                   |
 | meta-llama/Llama-3.1-8B              | 7.8B       |         32 |      4096 |        32 | silu              |    2048 |    128256 |      128 |   14336 | 8                   |
 | meta-llama/Llama-3.1-8B-Instruct     | 7.8B       |         32 |      4096 |        32 | silu              |    2048 |    128256 |      128 |   14336 | 8                   |
 | meta-llama/Llama-3.1-70B-Instruct    | 78B        |         80 |      8192 |        64 | silu              |    2048 |    128256 |      128 |   28672 | 8                   |
+| meta-llama/Llama-3.2-1B              | 1.1B       |         16 |      2048 |        32 | silu              |    2048 |    128256 |       64 |    8192 | 8                   |
+| meta-llama/Llama-3.2-3B              | 3.2B       |         28 |      3072 |        24 | silu              |    2048 |    128256 |      128 |    8192 | 8                   |
+| meta-llama/Llama-3.2-1B-Instruct     | 1.1B       |         16 |      2048 |        32 | silu              |    2048 |    128256 |       64 |    8192 | 8                   |
+| meta-llama/Llama-3.2-3B-Instruct     | 3.2B       |         28 |      3072 |        24 | silu              |    2048 |    128256 |      128 |    8192 | 8                   |
+| meta-llama/Llama-3.3-70B-Instruct    | 78B        |         80 |      8192 |        64 | silu              |    2048 |    128256 |      128 |   28672 | 8                   |
 | othello-gpt                          | 25M        |          8 |       512 |         8 | gelu              |      59 |        61 |       64 |    2048 |                     |
 | bert-base-cased                      | 85M        |         12 |       768 |        12 | gelu              |     512 |     28996 |       64 |    3072 |                     |
 | tiny-stories-1M                      | 393K       |          8 |        64 |        16 | gelu              |    2048 |     50257 |        4 |     256 |                     |

diff --git a/_static/coverage/d_37285d613390727b_can_be_used_as_mlp_py.html b/_static/coverage/d_37285d613390727b_can_be_used_as_mlp_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_gated_mlp_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -166,7 +166,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_gated_mlp_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>

diff --git a/_static/coverage/d_37285d613390727b_gated_mlp_4bit_py.html b/_static/coverage/d_37285d613390727b_gated_mlp_4bit_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_mlp_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -168,7 +168,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_mlp_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>

diff --git a/_static/coverage/d_37285d613390727b_gated_mlp_py.html b/_static/coverage/d_37285d613390727b_gated_mlp_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_gated_mlp_4bit_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -164,7 +164,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_gated_mlp_4bit_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>

diff --git a/_static/coverage/d_37285d613390727b_mlp_py.html b/_static/coverage/d_37285d613390727b_mlp_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_moe_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -140,7 +140,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_moe_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>

diff --git a/_static/coverage/d_37285d613390727b_moe_py.html b/_static/coverage/d_37285d613390727b_moe_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_pos_embed_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -204,7 +204,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_pos_embed_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>

diff --git a/_static/coverage/d_65d4430f90bfb219_activation_function_factory_py.html b/_static/coverage/d_65d4430f90bfb219_activation_function_factory_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_65d4430f90bfb219_mlp_factory_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -128,7 +128,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_65d4430f90bfb219_mlp_factory_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>

diff --git a/_static/coverage/d_65d4430f90bfb219_mlp_factory_py.html b/_static/coverage/d_65d4430f90bfb219_mlp_factory_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_head_detector_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -112,7 +112,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_head_detector_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>

diff --git a/_static/coverage/d_712808f24eb400fe___init___py.html b/_static/coverage/d_712808f24eb400fe___init___py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7___init___py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -91,7 +91,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7___init___py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>

diff --git a/_static/coverage/d_af97b5493da09a14_ActivationCache_py.html b/_static/coverage/d_af97b5493da09a14_ActivationCache_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_FactoredMatrix_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -1196,7 +1196,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_FactoredMatrix_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>

diff --git a/_static/coverage/d_af97b5493da09a14_FactoredMatrix_py.html b/_static/coverage/d_af97b5493da09a14_FactoredMatrix_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_HookedEncoder_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -365,7 +365,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_HookedEncoder_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>

diff --git a/_static/coverage/d_af97b5493da09a14_HookedEncoderDecoder_py.html b/_static/coverage/d_af97b5493da09a14_HookedEncoderDecoder_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_HookedTransformer_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -507,7 +507,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_HookedTransformer_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>

diff --git a/_static/coverage/d_af97b5493da09a14_HookedEncoder_py.html b/_static/coverage/d_af97b5493da09a14_HookedEncoder_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_HookedEncoderDecoder_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -461,7 +461,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_HookedEncoderDecoder_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>

diff --git a/_static/coverage/d_af97b5493da09a14_HookedTransformerConfig_py.html b/_static/coverage/d_af97b5493da09a14_HookedTransformerConfig_py.html
@@ -2,7 +2,7 @@
 <html>
 <head>
     <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
-    <title>Coverage for transformer_lens/HookedTransformerConfig.py: 91%</title>
+    <title>Coverage for transformer_lens/HookedTransformerConfig.py: 92%</title>
     <link rel="icon" sizes="32x32" href="favicon_32.png">
     <link rel="stylesheet" href="style.css" type="text/css">
     <script type="text/javascript" src="coverage_html.js" defer></script>
@@ -12,7 +12,7 @@
     <div class="content">
         <h1>
             <span class="text">Coverage for </span><b>transformer_lens/HookedTransformerConfig.py</b>:
-            <span class="pc_cov">91%</span>
+            <span class="pc_cov">92%</span>
         </h1>
         <aside id="help_panel_wrapper">
             <input id="help_panel_state" type="checkbox">
@@ -59,15 +59,15 @@ <h2>
             <button type="button" class="run button_toggle_run" value="run" data-shortcut="r" title="Toggle lines run">127<span class="text"> run</span></button>
             <button type="button" class="mis show_mis button_toggle_mis" value="mis" data-shortcut="m" title="Toggle lines missing">8<span class="text"> missing</span></button>
             <button type="button" class="exc show_exc button_toggle_exc" value="exc" data-shortcut="x" title="Toggle lines excluded">0<span class="text"> excluded</span></button>
-            <button type="button" class="par run show_par button_toggle_par" value="par" data-shortcut="p" title="Toggle lines partially run">7<span class="text"> partial</span></button>
+            <button type="button" class="par run show_par button_toggle_par" value="par" data-shortcut="p" title="Toggle lines partially run">6<span class="text"> partial</span></button>
         </h2>
         <p class="text">
             <a id="prevFileLink" class="nav" href="d_af97b5493da09a14_HookedTransformer_py.html">&#xab; prev</a> &nbsp; &nbsp;
             <a id="indexLink" class="nav" href="index.html">&Hat; index</a> &nbsp; &nbsp;
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_SVDInterpreter_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -373,7 +373,7 @@ <h2>
     <p class="run"><span class="n"><a id="t289" href="#t289">289</a></span><span class="t">            <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t290" href="#t290">290</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">act_fn</span> <span class="key">in</span> <span class="nam">SUPPORTED_ACTIVATIONS</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t291" href="#t291">291</a></span><span class="t">            <span class="op">)</span><span class="op">,</span> <span class="str">f"act_fn={self.act_fn} must be one of {SUPPORTED_ACTIVATIONS}"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t292" href="#t292">292</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">initializer_range</span> <span class="op">&lt;</span> <span class="num">0</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">init_mode</span> <span class="op">==</span> <span class="str">"gpt2"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">292&#x202F;&#x219B;&#x202F;295</span><span class="annotate long">line 292 didn't jump to line 295, because the condition on line 292 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t292" href="#t292">292</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">initializer_range</span> <span class="op">&lt;</span> <span class="num">0</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">init_mode</span> <span class="op">==</span> <span class="str">"gpt2"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t293" href="#t293">293</a></span><span class="t">            <span class="com"># Roughly copy the GPT-2 value, but proportional to sqrt(1/d_model)</span>&nbsp;</span><span class="r"></span></p>
     <p class="run"><span class="n"><a id="t294" href="#t294">294</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">initializer_range</span> <span class="op">=</span> <span class="num">0.8</span> <span class="op">/</span> <span class="nam">np</span><span class="op">.</span><span class="nam">sqrt</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">d_model</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="par run show_par"><span class="n"><a id="t295" href="#t295">295</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">initializer_range</span> <span class="op">&lt;</span> <span class="num">0</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">init_mode</span> <span class="op">!=</span> <span class="str">"gpt2"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">295&#x202F;&#x219B;&#x202F;297</span><span class="annotate long">line 295 didn't jump to line 297, because the condition on line 295 was never true</span></span></p>
@@ -463,7 +463,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_SVDInterpreter_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-12-14 00:54 +0000
+            created at 2024-12-31 02:13 +0000
         </p>
     </div>
 </footer>