Deploying to gh-pages from @ 30c90f4 🚀

TransformerLensOrg · Dec 14, 2024 · c67f384 · c67f384
1 parent b18a01d
commit c67f384
Show file tree

Hide file tree

Showing 80 changed files with 9,380 additions and 9,063 deletions.
diff --git a/_sources/generated/demos/Main_Demo.ipynb.txt b/_sources/generated/demos/Main_Demo.ipynb.txt
@@ -429,6 +429,26 @@
                 "cv.attention.attention_patterns(tokens=gpt2_str_tokens, attention=attention_pattern)"
             ]
         },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "In this case, we only wanted the layer 0 attention patterns, but we are storing the internal activations from all locations in the model. It's convenient to have access to all activations, but this can be prohibitively expensive for memory use with larger models, batch sizes, or sequence lengths. In addition, we don't need to do the full forward pass through the model to collect layer 0 attention patterns. The following cell will collect only the layer 0 attention patterns and stop the forward pass at layer 1, requiring far less memory and compute."
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "attn_hook_name = \"blocks.0.attn.hook_pattern\"\n",
+                "attn_layer = 0\n",
+                "_, gpt2_attn_cache = model.run_with_cache(gpt2_tokens, remove_batch_dim=True, stop_at_layer=attn_layer + 1, names_filter=[attn_hook_name])\n",
+                "gpt2_attn = gpt2_attn_cache[attn_hook_name]\n",
+                "assert torch.equal(gpt2_attn, attention_pattern)"
+            ]
+        },
         {
             "attachments": {},
             "cell_type": "markdown",

diff --git a/_sources/generated/model_properties_table.md.txt b/_sources/generated/model_properties_table.md.txt
@@ -170,6 +170,21 @@
 | Qwen/Qwen2-1.5B-Instruct             | 1.4B       |         28 |      1536 |        12 | silu              |    2048 |    151936 |      128 |    8960 | 2                   |
 | Qwen/Qwen2-7B                        | 7.1B       |         28 |      3584 |        28 | silu              |    2048 |    152064 |      128 |   18944 | 4                   |
 | Qwen/Qwen2-7B-Instruct               | 7.1B       |         28 |      3584 |        28 | silu              |    2048 |    152064 |      128 |   18944 | 4                   |
+| Qwen/Qwen2.5-0.5B                    | 391M       |         24 |       896 |        14 | silu              |    2048 |    151936 |       64 |    4864 | 2                   |
+| Qwen/Qwen2.5-0.5B-Instruct           | 391M       |         24 |       896 |        14 | silu              |    2048 |    151936 |       64 |    4864 | 2                   |
+| Qwen/Qwen2.5-1.5B                    | 1.4B       |         28 |      1536 |        12 | silu              |    2048 |    151936 |      128 |    8960 | 2                   |
+| Qwen/Qwen2.5-1.5B-Instruct           | 1.4B       |         28 |      1536 |        12 | silu              |    2048 |    151936 |      128 |    8960 | 2                   |
+| Qwen/Qwen2.5-3B                      | 3.0B       |         36 |      2048 |        16 | silu              |    2048 |    151936 |      128 |   11008 | 2                   |
+| Qwen/Qwen2.5-3B-Instruct             | 3.0B       |         36 |      2048 |        16 | silu              |    2048 |    151936 |      128 |   11008 | 2                   |
+| Qwen/Qwen2.5-7B                      | 7.1B       |         28 |      3584 |        28 | silu              |    2048 |    152064 |      128 |   18944 | 4                   |
+| Qwen/Qwen2.5-7B-Instruct             | 7.1B       |         28 |      3584 |        28 | silu              |    2048 |    152064 |      128 |   18944 | 4                   |
+| Qwen/Qwen2.5-14B                     | 15B        |         48 |      5120 |        40 | silu              |    2048 |    152064 |      128 |   13824 | 8                   |
+| Qwen/Qwen2.5-14B-Instruct            | 15B        |         48 |      5120 |        40 | silu              |    2048 |    152064 |      128 |   13824 | 8                   |
+| Qwen/Qwen2.5-32B                     | 34B        |         64 |      5120 |        40 | silu              |    2048 |    152064 |      128 |   27648 | 8                   |
+| Qwen/Qwen2.5-32B-Instruct            | 34B        |         64 |      5120 |        40 | silu              |    2048 |    152064 |      128 |   27648 | 8                   |
+| Qwen/Qwen2.5-72B                     | 80B        |         80 |      8192 |        64 | silu              |    2048 |    152064 |      128 |   29568 | 8                   |
+| Qwen/Qwen2.5-72B-Instruct            | 80B        |         80 |      8192 |        64 | silu              |    2048 |    152064 |      128 |   29568 | 8                   |
+| Qwen/QwQ-32B-Preview                 | 34B        |         64 |      5120 |        40 | silu              |    2048 |    152064 |      128 |   27648 | 8                   |
 | phi-1                                | 1.2B       |         24 |      2048 |        32 | gelu              |    2048 |     51200 |       64 |    8192 |                     |
 | phi-1_5                              | 1.2B       |         24 |      2048 |        32 | gelu              |    2048 |     51200 |       64 |    8192 |                     |
 | phi-2                                | 2.5B       |         32 |      2560 |        32 | gelu              |    2048 |     51200 |       80 |   10240 |                     |

diff --git a/_static/coverage/d_37285d613390727b_can_be_used_as_mlp_py.html b/_static/coverage/d_37285d613390727b_can_be_used_as_mlp_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_gated_mlp_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -166,7 +166,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_gated_mlp_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>

diff --git a/_static/coverage/d_37285d613390727b_gated_mlp_4bit_py.html b/_static/coverage/d_37285d613390727b_gated_mlp_4bit_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_mlp_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -168,7 +168,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_mlp_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>

diff --git a/_static/coverage/d_37285d613390727b_gated_mlp_py.html b/_static/coverage/d_37285d613390727b_gated_mlp_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_gated_mlp_4bit_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -164,7 +164,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_gated_mlp_4bit_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>

diff --git a/_static/coverage/d_37285d613390727b_mlp_py.html b/_static/coverage/d_37285d613390727b_mlp_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_moe_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -140,7 +140,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_moe_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>

diff --git a/_static/coverage/d_37285d613390727b_moe_py.html b/_static/coverage/d_37285d613390727b_moe_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_pos_embed_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -204,7 +204,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_pos_embed_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>

diff --git a/_static/coverage/d_65d4430f90bfb219_activation_function_factory_py.html b/_static/coverage/d_65d4430f90bfb219_activation_function_factory_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_65d4430f90bfb219_mlp_factory_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -128,7 +128,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_65d4430f90bfb219_mlp_factory_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>

diff --git a/_static/coverage/d_65d4430f90bfb219_mlp_factory_py.html b/_static/coverage/d_65d4430f90bfb219_mlp_factory_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_head_detector_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -112,7 +112,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_head_detector_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>

diff --git a/_static/coverage/d_712808f24eb400fe___init___py.html b/_static/coverage/d_712808f24eb400fe___init___py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7___init___py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -91,7 +91,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7___init___py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>