From ca6b8db53a07e7b729112795861fcd113bfe6cac Mon Sep 17 00:00:00 2001 From: Connor Kissane <67170576+ckkissane@users.noreply.github.com> Date: Tue, 30 Apr 2024 08:20:23 -0400 Subject: [PATCH] HookedSAETransformer (#536) * implement HookedSAETransformer * clean up imports * apply format * only recompute error if use_error_term * add tests * run format * fix import * match to hooks API * improve doc strings * improve demo * address Arthur feedback * try to fix indent: * try to fix indent again * change doc code block --- README.md | 3 +- demos/HookedSAETransformerDemo.ipynb | 18616 ++++++++++++++++++++ tests/unit/test_hooked_sae.py | 191 + tests/unit/test_hooked_sae_transformer.py | 515 + transformer_lens/HookedSAE.py | 118 + transformer_lens/HookedSAEConfig.py | 64 + transformer_lens/HookedSAETransformer.py | 290 + transformer_lens/__init__.py | 3 + 8 files changed, 19799 insertions(+), 1 deletion(-) create mode 100644 demos/HookedSAETransformerDemo.ipynb create mode 100644 tests/unit/test_hooked_sae.py create mode 100644 tests/unit/test_hooked_sae_transformer.py create mode 100644 transformer_lens/HookedSAE.py create mode 100644 transformer_lens/HookedSAEConfig.py create mode 100644 transformer_lens/HookedSAETransformer.py diff --git a/README.md b/README.md index 3f65c881d..ab7d11396 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ TransformerLens lets you load in 50+ different open source language models, and activations of the model to you. You can cache any internal activation in the model, and add in functions to edit, remove or replace these activations as the model runs. -~~ [OCTOBER SURVEY HERE](https://forms.gle/bw7U3PfioacDtFmT8) ~~ +The library also now supports mechanistic interpretability with SAEs (sparse autoencoders)! With [HookedSAETransformer](https://colab.research.google.com/github/neelnanda-io/TransformerLens/blob/hooked-sae-transformer/demos/HookedSAETransformerDemo.ipynb), you can splice in SAEs during inference and cache + intervene on SAE activations. We recommend [SAELens](https://github.com/jbloomAus/SAELens) (built on top of TransformerLens) for training SAEs. ## Quick Start @@ -51,6 +51,7 @@ logits, activations = model.run_with_cache("Hello World") * [Introduction to the Library and Mech Interp](https://arena-ch1-transformers.streamlit.app/[1.2]_Intro_to_Mech_Interp) * [Demo of Main TransformerLens Features](https://neelnanda.io/transformer-lens-demo) +* [Demo of HookedSAETransformer Features](https://colab.research.google.com/github/neelnanda-io/TransformerLens/blob/hooked-sae-transformer/demos/HookedSAETransformerDemo.ipynb) ## Gallery diff --git a/demos/HookedSAETransformerDemo.ipynb b/demos/HookedSAETransformerDemo.ipynb new file mode 100644 index 000000000..77d0d7c37 --- /dev/null +++ b/demos/HookedSAETransformerDemo.ipynb @@ -0,0 +1,18616 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + " \"Open\n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# HookedSAETransformer Demo" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "HookedSAETransformer is a lightweight extension of HookedTransformer that allows you to \"splice in\" Sparse Autoencoders. This makes it easy to do exploratory analysis such as: running inference with SAEs attached, caching SAE feature activations, and intervening on SAE activations with hooks.\n", + "\n", + "I (Connor Kissane) implemented this to accelerate research on [Attention SAEs](https://www.lesswrong.com/posts/DtdzGwFh9dCfsekZZ/sparse-autoencoders-work-on-attention-layer-outputs) based on suggestions from Arthur Conmy and Neel Nanda, and found that it was well worth the time and effort. I hope other researchers will also find the library useful! This notebook demonstrates how it works and how to use it." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setup" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Running as a Jupyter notebook - intended for development only!\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/tmp/ipykernel_10435/2185356984.py:16: DeprecationWarning: `magic(...)` is deprecated since IPython 0.13 (warning added in 8.1), use run_line_magic(magic_name, parameter_s).\n", + " ipython.magic(\"load_ext autoreload\")\n", + "/tmp/ipykernel_10435/2185356984.py:17: DeprecationWarning: `magic(...)` is deprecated since IPython 0.13 (warning added in 8.1), use run_line_magic(magic_name, parameter_s).\n", + " ipython.magic(\"autoreload 2\")\n" + ] + } + ], + "source": [ + "# Janky code to do different setup when run in a Colab notebook vs VSCode\n", + "DEVELOPMENT_MODE = False\n", + "try:\n", + " import google.colab\n", + " IN_COLAB = True\n", + " print(\"Running as a Colab notebook\")\n", + " %pip install git+https://github.com/ckkissane/TransformerLens@hooked-sae-transformer\n", + " \n", + "except:\n", + " IN_COLAB = False\n", + " print(\"Running as a Jupyter notebook - intended for development only!\")\n", + " from IPython import get_ipython\n", + "\n", + " ipython = get_ipython()\n", + " # Code to automatically update the HookedTransformer code as its edited without restarting the kernel\n", + " ipython.magic(\"load_ext autoreload\")\n", + " ipython.magic(\"autoreload 2\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "import transformer_lens.utils as utils\n", + "\n", + "import plotly.express as px\n", + "import tqdm\n", + "from functools import partial\n", + "import einops\n", + "import plotly.graph_objects as go\n", + "\n", + "update_layout_set = {\n", + " \"xaxis_range\", \"yaxis_range\", \"hovermode\", \"xaxis_title\", \"yaxis_title\", \"colorbar\", \"colorscale\", \"coloraxis\",\n", + " \"title_x\", \"bargap\", \"bargroupgap\", \"xaxis_tickformat\", \"yaxis_tickformat\", \"title_y\", \"legend_title_text\", \"xaxis_showgrid\",\n", + " \"xaxis_gridwidth\", \"xaxis_gridcolor\", \"yaxis_showgrid\", \"yaxis_gridwidth\"\n", + "}\n", + "\n", + "def imshow(tensor, renderer=None, xaxis=\"\", yaxis=\"\", **kwargs):\n", + " if isinstance(tensor, list):\n", + " tensor = torch.stack(tensor)\n", + " kwargs_post = {k: v for k, v in kwargs.items() if k in update_layout_set}\n", + " kwargs_pre = {k: v for k, v in kwargs.items() if k not in update_layout_set}\n", + " if \"facet_labels\" in kwargs_pre:\n", + " facet_labels = kwargs_pre.pop(\"facet_labels\")\n", + " else:\n", + " facet_labels = None\n", + " if \"color_continuous_scale\" not in kwargs_pre:\n", + " kwargs_pre[\"color_continuous_scale\"] = \"RdBu\"\n", + " fig = px.imshow(utils.to_numpy(tensor), color_continuous_midpoint=0.0,labels={\"x\":xaxis, \"y\":yaxis}, **kwargs_pre).update_layout(**kwargs_post)\n", + " if facet_labels:\n", + " for i, label in enumerate(facet_labels):\n", + " fig.layout.annotations[i]['text'] = label\n", + "\n", + " fig.show(renderer)\n", + "\n", + "def scatter(x, y, xaxis=\"\", yaxis=\"\", caxis=\"\", renderer=None, return_fig=False, **kwargs):\n", + " x = utils.to_numpy(x)\n", + " y = utils.to_numpy(y)\n", + " fig = px.scatter(y=y, x=x, labels={\"x\":xaxis, \"y\":yaxis, \"color\":caxis}, **kwargs)\n", + " if return_fig:\n", + " return fig\n", + " fig.show(renderer)\n", + "\n", + "from typing import List\n", + "def show_avg_logit_diffs(x_axis: List[str], per_prompt_logit_diffs: List[torch.tensor]):\n", + "\n", + "\n", + " y_data = [per_prompt_logit_diff.mean().item() for per_prompt_logit_diff in per_prompt_logit_diffs]\n", + " error_y_data = [per_prompt_logit_diff.std().item() for per_prompt_logit_diff in per_prompt_logit_diffs] \n", + "\n", + " fig = go.Figure(data=[go.Bar(\n", + " x=x_axis,\n", + " y=y_data,\n", + " error_y=dict(\n", + " type='data', # specifies that the actual values are given\n", + " array=error_y_data, # the magnitudes of the errors\n", + " visible=True # make error bars visible\n", + " ),\n", + " )])\n", + "\n", + " # Customize layout\n", + " fig.update_layout(title_text=f'Logit Diff after Interventions',\n", + " xaxis_title_text='Intervention',\n", + " yaxis_title_text='Logit diff',\n", + " plot_bgcolor='white')\n", + "\n", + " # Show the figure\n", + " fig.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", + "torch.set_grad_enabled(False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Loading and Running Models" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Just like a [HookedTransformer](https://neelnanda-io.github.io/TransformerLens/generated/demos/Main_Demo.html#Loading-and-Running-Models), we can load in any model that's supported in TransformerLens with the `HookedSAETransformer.from_pretrained(MODEL_NAME)`. In this demo we'll use GPT-2 small." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using pad_token, but it is not set yet.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loaded pretrained model gpt2-small into HookedTransformer\n", + "Moving model to device: cuda\n" + ] + } + ], + "source": [ + "from transformer_lens import HookedSAETransformer\n", + "model: HookedSAETransformer = HookedSAETransformer.from_pretrained(\"gpt2-small\").to(device)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "By default HookedSAETransformer will behave exactly like a HookedTransformer. We'll explore the main features of HookedSAETransformer on the classic IOI task, so let's first sanity check that GPT2-small can do the IOI task without any SAEs attached:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['When John and Mary went to the shops, Mary gave the bag to', 'When John and Mary went to the shops, John gave the bag to', 'When Tom and James went to the park, James gave the ball to', 'When Tom and James went to the park, Tom gave the ball to', 'When Dan and Sid went to the shops, Sid gave an apple to', 'When Dan and Sid went to the shops, Dan gave an apple to', 'After Martin and Amy went to the park, Amy gave a drink to', 'After Martin and Amy went to the park, Martin gave a drink to']\n", + "[(' John', ' Mary'), (' Mary', ' John'), (' Tom', ' James'), (' James', ' Tom'), (' Dan', ' Sid'), (' Sid', ' Dan'), (' Martin', ' Amy'), (' Amy', ' Martin')]\n" + ] + } + ], + "source": [ + "prompt_format = [\n", + " \"When John and Mary went to the shops,{} gave the bag to\",\n", + " \"When Tom and James went to the park,{} gave the ball to\",\n", + " \"When Dan and Sid went to the shops,{} gave an apple to\",\n", + " \"After Martin and Amy went to the park,{} gave a drink to\",\n", + "]\n", + "names = [\n", + " (\" John\", \" Mary\",),\n", + " (\" Tom\", \" James\"),\n", + " (\" Dan\", \" Sid\"),\n", + " (\" Martin\", \" Amy\"),\n", + "]\n", + "# List of prompts\n", + "prompts = []\n", + "# List of answers, in the format (correct, incorrect)\n", + "answers = []\n", + "# List of the token (ie an integer) corresponding to each answer, in the format (correct_token, incorrect_token)\n", + "answer_tokens = []\n", + "for i in range(len(prompt_format)):\n", + " for j in range(2):\n", + " answers.append((names[i][j], names[i][1 - j]))\n", + " answer_tokens.append(\n", + " (\n", + " model.to_single_token(answers[-1][0]),\n", + " model.to_single_token(answers[-1][1]),\n", + " )\n", + " )\n", + " # Insert the *incorrect* answer to the prompt, making the correct answer the indirect object.\n", + " prompts.append(prompt_format[i].format(answers[-1][1]))\n", + "answer_tokens = torch.tensor(answer_tokens).to(device)\n", + "print(prompts)\n", + "print(answers)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Original average logit diff: 3.5518884658813477\n", + "Original per prompt logit diff: tensor([3.2016, 3.3367, 2.7095, 3.7975, 1.7204, 5.2812, 2.6008, 5.7674],\n", + " device='cuda:0')\n" + ] + } + ], + "source": [ + "def logits_to_ave_logit_diff(logits, answer_tokens, per_prompt=False):\n", + " # Only the final logits are relevant for the answer\n", + " final_logits = logits[:, -1, :]\n", + " answer_logits = final_logits.gather(dim=-1, index=answer_tokens)\n", + " answer_logit_diff = answer_logits[:, 0] - answer_logits[:, 1]\n", + " if per_prompt:\n", + " return answer_logit_diff\n", + " else:\n", + " return answer_logit_diff.mean()\n", + " \n", + "tokens = model.to_tokens(prompts, prepend_bos=True)\n", + "original_logits, cache = model.run_with_cache(tokens)\n", + "original_average_logit_diff = logits_to_ave_logit_diff(original_logits, answer_tokens)\n", + "print(f\"Original average logit diff: {original_average_logit_diff}\")\n", + "original_per_prompt_logit_diff = logits_to_ave_logit_diff(original_logits, answer_tokens, per_prompt=True)\n", + "print(f\"Original per prompt logit diff: {original_per_prompt_logit_diff}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# HookedSAEs" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In order to use the key features of HookedSAETransformer, we first need to load in SAEs.\n", + "\n", + "HookedSAE is an SAE class we've implemented to have TransformerLens hooks around the SAE activations. While we will use it out of the box, it is designed to be hackable: you can copy and paste the HookedSAE class into a notebook and completely change the architecture / hook names, and as long as it reconstructs the activations, it should still work.\n", + "\n", + "You can initialize a HookedSAE with a HookedSAEConfig:\n", + "```\n", + "cfg = HookedSAEConfig(\n", + " d_sae (int): The size of the dictionary.\n", + " d_in (int): The dimension of the input activations for the SAE\n", + " hook_name (str): The hook name of the activation the SAE was trained on (eg. blocks.0.attn.hook_z)\n", + ")\n", + "hooked_sae = HookedSAE(cfg)\n", + "```\n", + "\n", + "Note you'll likely have to write some basic conversion code to match configs / state dicts to the HookedSAE when loading in an open sourced SAE (eg from HuggingFace). We'll use our GPT-2 Small [Attention SAEs](https://www.alignmentforum.org/posts/FSTRedtjuHa4Gfdbr/attention-saes-scale-to-gpt-2-small) to demonstrate. For convenience, we'll load in all of our attention SAEs from HuggingFace, convert them to HookedSAEs, and store them in a dictionary that maps each hook_name (str) to the corresponding HookedSAE.\n", + "\n", + "
\n", + "\n", + "Later we'll show how to add HookedSAEs to the HookedSAETransformer (replacing model activations with their SAE reconstructions). When you add a HookedSAE, HookedSAETransformer just treats this a black box that takes some activation as an input, and outputs a tensor of the same shape. \n", + "\n", + "With this in mind, the HookedSAE is designed to be simple and hackable. Think of it as a convenient default class that you can copy and edit. As long as it takes a TransformerLens activation as input, and outputs a tensor of the same shape, you should be able to add it to your HookedSAETransformer.\n", + "\n", + "You probably don't even need to use the HookedSAE class, although it's recommended. The sae can be any pytorch module that takes in some activation at hook_name and outputs a tensor of the same shape. The two assumptions that HookedSAETransformer makes when adding SAEs are:\n", + "1. The SAE class has a cfg attribute, sae.cfg.hook_name (str), for the activation that the SAE was trained to reconstruct (in TransformerLens notation e.g. 'blocks.0.attn.hook_z')\n", + "2. The SAE takes that activation as input, and outputs a tensor of the same shape.\n", + "\n", + "The main benefit of HookedSAE is that it's a subclass of HookedRootModule, so we can add hooks to SAE activations. This makes it easy to leverage existing TL functionality like run_with_cache and run_with_hooks with SAEs.\n", + "\n", + "
" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "dict_keys(['blocks.0.attn.hook_z', 'blocks.1.attn.hook_z', 'blocks.2.attn.hook_z', 'blocks.3.attn.hook_z', 'blocks.4.attn.hook_z', 'blocks.5.attn.hook_z', 'blocks.6.attn.hook_z', 'blocks.7.attn.hook_z', 'blocks.8.attn.hook_z', 'blocks.9.attn.hook_z', 'blocks.10.attn.hook_z', 'blocks.11.attn.hook_z'])\n" + ] + } + ], + "source": [ + "from transformer_lens import HookedSAE, HookedSAEConfig\n", + "from transformer_lens.utils import download_file_from_hf\n", + "def attn_sae_cfg_to_hooked_sae_cfg(attn_sae_cfg):\n", + " new_cfg = {\n", + " \"d_sae\": attn_sae_cfg[\"dict_size\"],\n", + " \"d_in\": attn_sae_cfg[\"act_size\"],\n", + " \"hook_name\": attn_sae_cfg[\"act_name\"],\n", + " }\n", + " return HookedSAEConfig.from_dict(new_cfg)\n", + "\n", + "auto_encoder_runs = [\n", + " \"gpt2-small_L0_Hcat_z_lr1.20e-03_l11.80e+00_ds24576_bs4096_dc1.00e-06_rsanthropic_rie25000_nr4_v9\",\n", + " \"gpt2-small_L1_Hcat_z_lr1.20e-03_l18.00e-01_ds24576_bs4096_dc1.00e-06_rsanthropic_rie25000_nr4_v5\",\n", + " \"gpt2-small_L2_Hcat_z_lr1.20e-03_l11.00e+00_ds24576_bs4096_dc1.00e-06_rsanthropic_rie25000_nr4_v4\",\n", + " \"gpt2-small_L3_Hcat_z_lr1.20e-03_l19.00e-01_ds24576_bs4096_dc1.00e-06_rsanthropic_rie25000_nr4_v9\",\n", + " \"gpt2-small_L4_Hcat_z_lr1.20e-03_l11.10e+00_ds24576_bs4096_dc1.00e-06_rsanthropic_rie25000_nr4_v7\",\n", + " \"gpt2-small_L5_Hcat_z_lr1.20e-03_l11.00e+00_ds49152_bs4096_dc1.00e-06_rsanthropic_rie25000_nr4_v9\",\n", + " \"gpt2-small_L6_Hcat_z_lr1.20e-03_l11.10e+00_ds24576_bs4096_dc1.00e-06_rsanthropic_rie25000_nr4_v9\",\n", + " \"gpt2-small_L7_Hcat_z_lr1.20e-03_l11.10e+00_ds49152_bs4096_dc1.00e-06_rsanthropic_rie25000_nr4_v9\",\n", + " \"gpt2-small_L8_Hcat_z_lr1.20e-03_l11.30e+00_ds24576_bs4096_dc1.00e-05_rsanthropic_rie25000_nr4_v6\",\n", + " \"gpt2-small_L9_Hcat_z_lr1.20e-03_l11.20e+00_ds24576_bs4096_dc1.00e-06_rsanthropic_rie25000_nr4_v9\",\n", + " \"gpt2-small_L10_Hcat_z_lr1.20e-03_l11.30e+00_ds24576_bs4096_dc1.00e-05_rsanthropic_rie25000_nr4_v9\",\n", + " \"gpt2-small_L11_Hcat_z_lr1.20e-03_l13.00e+00_ds24576_bs4096_dc3.16e-06_rsanthropic_rie25000_nr4_v9\",\n", + "]\n", + "\n", + "hf_repo = \"ckkissane/attn-saes-gpt2-small-all-layers\"\n", + "\n", + "hook_name_to_sae = {}\n", + "for auto_encoder_run in auto_encoder_runs:\n", + " attn_sae_cfg = download_file_from_hf(hf_repo, f\"{auto_encoder_run}_cfg.json\")\n", + " cfg = attn_sae_cfg_to_hooked_sae_cfg(attn_sae_cfg)\n", + " \n", + " state_dict = download_file_from_hf(hf_repo, f\"{auto_encoder_run}.pt\", force_is_torch=True)\n", + " \n", + " hooked_sae = HookedSAE(cfg)\n", + " hooked_sae.load_state_dict(state_dict)\n", + " \n", + " hook_name_to_sae[cfg.hook_name] = hooked_sae\n", + "print(hook_name_to_sae.keys())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Run with SAEs" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The key feature of HookedSAETransformer is being able to \"splice in\" SAEs, replacing model activations with their SAE reconstructions. \n", + "\n", + "To run a forward pass with SAEs attached use `model.run_with_saes(tokens, saes=saes)`, where saes is a list of HookedSAEs that you want to add for just this forward pass. These will be reset immediately after the forward pass, returning the model to its original state.\n", + "\n", + "I expect this to be particularly useful for evaluating SAEs (eg [Gurnee](https://www.alignmentforum.org/posts/rZPiuFxESMxCDHe4B/sae-reconstruction-errors-are-empirically-pathological)), including evaluating how SAE reconstructions affect the models ability to perform certain tasks (eg [Makelov et al.](https://openreview.net/forum?id=MHIX9H8aYF&referrer=%5Bthe%20profile%20of%20Neel%20Nanda%5D(%2Fprofile%3Fid%3D~Neel_Nanda1)))\n", + "\n", + "To demonstrate, let's use `run_with_saes` to evaluate many combinations of SAEs on different cross sections of the IOI circuit.\n", + "\n", + "
\n", + "\n", + "Under the hood, TransformerLens already wraps activations with a HookPoint object. HookPoint is a dummy pytorch module that acts as an identity function by default, and is only used to access the activation with PyTorch hooks. When you run_with_saes, HookedSAETransformer temporarily replaces these HookPoints with the given HookedSAEs, which take the activation as input and replace it with the HookedSAE output (the reconstructed activation) during the forward pass. \n", + "\n", + "Since HookedSAE is a subclass of HookedRootModule, we also are able to add PyTorch hooks to the corresponding SAE activations, as we'll use later.\n", + "\n", + "
" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + " \n", + " " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.plotly.v1+json": { + "config": { + "plotlyServerURL": "https://plot.ly" + }, + "data": [ + { + "error_y": { + "array": [ + 1.3678313493728638, + 1.6846193075180054, + 1.3839112520217896, + 1.6782633066177368, + 0.8939867615699768, + 2.2888872623443604 + ], + "type": "data", + "visible": true + }, + "type": "bar", + "x": [ + "Clean Baseline", + "With SAEs L[0, 3]", + "With SAEs L[2, 4]", + "With SAEs L[5, 6]", + "With SAEs L[7, 8]", + "With SAEs L[9, 10, 11]" + ], + "y": [ + 3.5518884658813477, + 2.580843925476074, + 3.3641157150268555, + 3.3500614166259766, + 1.5024915933609009, + 7.072007179260254 + ] + } + ], + "layout": { + "plot_bgcolor": "white", + "template": { + "data": { + "bar": [ + { + "error_x": { + "color": "#2a3f5f" + }, + "error_y": { + "color": "#2a3f5f" + }, + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "bar" + } + ], + "barpolar": [ + { + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "barpolar" + } + ], + "carpet": [ + { + "aaxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "baxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "type": "carpet" + } + ], + "choropleth": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "choropleth" + } + ], + "contour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "contour" + } + ], + "contourcarpet": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "contourcarpet" + } + ], + "heatmap": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmap" + } + ], + "heatmapgl": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmapgl" + } + ], + "histogram": [ + { + "marker": { + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "histogram" + } + ], + "histogram2d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2d" + } + ], + "histogram2dcontour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2dcontour" + } + ], + "mesh3d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "mesh3d" + } + ], + "parcoords": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "parcoords" + } + ], + "pie": [ + { + "automargin": true, + "type": "pie" + } + ], + "scatter": [ + { + "fillpattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + }, + "type": "scatter" + } + ], + "scatter3d": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatter3d" + } + ], + "scattercarpet": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattercarpet" + } + ], + "scattergeo": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergeo" + } + ], + "scattergl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergl" + } + ], + "scattermapbox": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermapbox" + } + ], + "scatterpolar": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolar" + } + ], + "scatterpolargl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolargl" + } + ], + "scatterternary": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterternary" + } + ], + "surface": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "surface" + } + ], + "table": [ + { + "cells": { + "fill": { + "color": "#EBF0F8" + }, + "line": { + "color": "white" + } + }, + "header": { + "fill": { + "color": "#C8D4E3" + }, + "line": { + "color": "white" + } + }, + "type": "table" + } + ] + }, + "layout": { + "annotationdefaults": { + "arrowcolor": "#2a3f5f", + "arrowhead": 0, + "arrowwidth": 1 + }, + "autotypenumbers": "strict", + "coloraxis": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "colorscale": { + "diverging": [ + [ + 0, + "#8e0152" + ], + [ + 0.1, + "#c51b7d" + ], + [ + 0.2, + "#de77ae" + ], + [ + 0.3, + "#f1b6da" + ], + [ + 0.4, + "#fde0ef" + ], + [ + 0.5, + "#f7f7f7" + ], + [ + 0.6, + "#e6f5d0" + ], + [ + 0.7, + "#b8e186" + ], + [ + 0.8, + "#7fbc41" + ], + [ + 0.9, + "#4d9221" + ], + [ + 1, + "#276419" + ] + ], + "sequential": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "sequentialminus": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ] + }, + "colorway": [ + "#636efa", + "#EF553B", + "#00cc96", + "#ab63fa", + "#FFA15A", + "#19d3f3", + "#FF6692", + "#B6E880", + "#FF97FF", + "#FECB52" + ], + "font": { + "color": "#2a3f5f" + }, + "geo": { + "bgcolor": "white", + "lakecolor": "white", + "landcolor": "#E5ECF6", + "showlakes": true, + "showland": true, + "subunitcolor": "white" + }, + "hoverlabel": { + "align": "left" + }, + "hovermode": "closest", + "mapbox": { + "style": "light" + }, + "paper_bgcolor": "white", + "plot_bgcolor": "#E5ECF6", + "polar": { + "angularaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "radialaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "scene": { + "xaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "yaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "zaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + } + }, + "shapedefaults": { + "line": { + "color": "#2a3f5f" + } + }, + "ternary": { + "aaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "baxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "caxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "title": { + "x": 0.05 + }, + "xaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + }, + "yaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + } + } + }, + "title": { + "text": "Logit Diff after Interventions" + }, + "xaxis": { + "title": { + "text": "Intervention" + } + }, + "yaxis": { + "title": { + "text": "Logit diff" + } + } + } + }, + "text/html": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "all_layers = [[0, 3], [2, 4], [5,6], [7, 8], [9, 10, 11]]\n", + "x_axis = ['Clean Baseline']\n", + "per_prompt_logit_diffs = [\n", + " original_per_prompt_logit_diff, \n", + "]\n", + "\n", + "for layers in all_layers:\n", + " hooked_saes = [hook_name_to_sae[utils.get_act_name('z', layer)] for layer in layers]\n", + " logits_with_saes = model.run_with_saes(tokens, saes=hooked_saes)\n", + " average_logit_diff_with_saes = logits_to_ave_logit_diff(logits_with_saes, answer_tokens)\n", + " per_prompt_diff_with_saes = logits_to_ave_logit_diff(logits_with_saes, answer_tokens, per_prompt=True)\n", + " \n", + " x_axis.append(f\"With SAEs L{layers}\")\n", + " per_prompt_logit_diffs.append(per_prompt_diff_with_saes)\n", + "\n", + "show_avg_logit_diffs(x_axis, per_prompt_logit_diffs)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Run with cache (with SAEs)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We often want to see what SAE features are active on a given prompt. With HookedSAETransformer, you can cache HookedSAE activations (and all the other standard activations) with `logits, cache = model.run_with_cache_with_saes(tokens, saes=saes)`. Just as `run_with_saes` is a wapper around the standard forward pass, `run_with_cache_with_saes` is a wrapper around `run_with_cache`, and will also only add these saes for one forward pass before returning the model to its original state. \n", + "\n", + "To access SAE activations from the cache, the corresponding hook names will generally be the HookedTransformer hook_name (eg blocks.5.attn.hook_z) + the hookedSAE hooked name preceeded by a period (eg .hook_sae_acts_post).\n", + "\n", + "`run_with_cache_with_saes` makes it easy to explore which SAE features are active across any input. Let's explore the active features at the S2 position for our L5 Attention SAE across all of our IOI prompts:" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.plotly.v1+json": { + "config": { + "plotlyServerURL": "https://plot.ly" + }, + "data": [ + { + "coloraxis": "coloraxis", + "hovertemplate": "Feature Id: %{x}
Prompt: %{y}
color: %{z}", + "name": "0", + "type": "heatmap", + "x": [ + "46", + "345", + "702", + "1372", + "1755", + "1965", + "2457", + "2496", + "2646", + "2999", + "3047", + "4569", + "5132", + "5203", + "5508", + "5940", + "6144", + "6371", + "6515", + "6558", + "6812", + "7092", + "7515", + "7907", + "8063", + "8623", + "8737", + "8768", + "9096", + "9102", + "9186", + "9463", + "9746", + "9913", + "10581", + "10894", + "12109", + "12485", + "12764", + "12866", + "13063", + "13624", + "13707", + "13777", + "14844", + "15050", + "15170", + "15696", + "16178", + "16892", + "17156", + "17259", + "17497", + "17854", + "18043", + "18210", + "18318", + "18385", + "18440", + "18920", + "19183", + "19263", + "19442", + "19524", + "19573", + "20838", + "21151", + "21657", + "22108", + "23578", + "24091", + "24217", + "25792", + "26373", + "26410", + "27535", + "27787", + "27811", + "27960", + "28061", + "28241", + "28242", + "28254", + "28349", + "28977", + "29027", + "29482", + "29603", + "29700", + "29822", + "32177", + "32920", + "33320", + "33730", + "33966", + "34177", + "34334", + "34947", + "35403", + "35425", + "35579", + "35665", + "35815", + "36109", + "36172", + "36451", + "36767", + "36917", + "38570", + "39962", + "40409", + "40418", + "40661", + "41162", + "41185", + "41552", + "42024", + "42161", + "42437", + "42577", + "42882", + "42931", + "43035", + "43414", + "43643", + "43662", + "44203", + "44256", + "44452", + "44652", + "45179", + "45814", + "45984", + "46880", + "47117", + "47170", + "47231", + "47313", + "47680", + "48063", + "48703" + ], + "xaxis": "x", + "yaxis": "y", + "z": [ + [ + 0.23392018675804138, + 0, + 0, + 0.04335343837738037, + 0.44275617599487305, + 0, + 0, + 0.07259953022003174, + 0, + 0.6985604763031006, + 1.262436866760254, + 0, + 0.04656928777694702, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.45666736364364624, + 0.10434150695800781, + 0.30980953574180603, + 0.3319076895713806, + 0, + 0, + 0, + 0, + 1.7836596965789795, + 0, + 0, + 0.142583429813385, + 0.046830952167510986, + 0.3180348575115204, + 0.2927079200744629, + 0.12267106771469116, + 2.5688514709472656, + 0.2917236089706421, + 0.12333670258522034, + 0, + 0.1778419017791748, + 0, + 0.023626387119293213, + 0.02943563461303711, + 0, + 0.048882365226745605, + 0.13625454902648926, + 0, + 0, + 0.2634885013103485, + 0, + 0, + 0, + 0.21662655472755432, + 0, + 0, + 0, + 0.06997489929199219, + 0.006345987319946289, + 0, + 0.16112494468688965, + 0.4190089702606201, + 0, + 2.3819468021392822, + 1.0431660413742065, + 0, + 0.08364987373352051, + 0, + 0, + 0.3451769948005676, + 0.7391350865364075, + 0.4456520080566406, + 0.0019606351852416992, + 0.39914217591285706, + 0, + 0, + 0, + 0.29958274960517883, + 0.44243645668029785, + 0, + 0.1259920299053192, + 0.8349504470825195, + 0.37993764877319336, + 0.2633737325668335, + 0.08324140310287476, + 0, + 0, + 0.10421907901763916, + 0, + 0, + 0, + 0.36972635984420776, + 0, + 0, + 0, + 0, + 0.5578295588493347, + 0, + 0.9233021140098572, + 0, + 0.10010790824890137, + 0, + 0.45082613825798035, + 0, + 0, + 0, + 0.21043556928634644, + 0.12981292605400085, + 0.11557984352111816, + 0, + 0, + 0.17571094632148743, + 0.2823787331581116, + 0.1122598648071289, + 0, + 0, + 0.012049257755279541, + 0, + 0, + 0, + 2.417463541030884, + 0.0547795295715332, + 0.05216425657272339, + 0, + 0.6592545509338379, + 0.003663182258605957, + 0, + 0, + 0.04937589168548584, + 0.025814831256866455, + 0, + 0.8019273281097412, + 0, + 0.10218703746795654 + ], + [ + 0, + 0, + 0.3230956792831421, + 0, + 0, + 0, + 0.026041746139526367, + 0.31818556785583496, + 0, + 0.4900796413421631, + 0.04911249876022339, + 0, + 0, + 0.07309412956237793, + 0.08089971542358398, + 0.17180073261260986, + 0, + 0, + 0, + 0, + 0, + 0, + 2.3956947326660156, + 0, + 0, + 0.15781426429748535, + 0, + 0.5073252320289612, + 0.21765804290771484, + 0, + 0, + 1.618570327758789, + 0, + 0.22485831379890442, + 0.0830467939376831, + 0.7055595517158508, + 0, + 0, + 0, + 0, + 0.23371747136116028, + 0, + 0, + 0.6983060240745544, + 0, + 0, + 0, + 0, + 0.30831730365753174, + 0, + 0.417669415473938, + 0.05292201042175293, + 0, + 0, + 0, + 1.3391070365905762, + 0, + 0.41352108120918274, + 0, + 0, + 0, + 0.037178993225097656, + 0, + 0, + 0, + 0, + 0.2702980041503906, + 0, + 0, + 0.18745100498199463, + 1.3330132961273193, + 0.5793700814247131, + 0.33893001079559326, + 0, + 0.11196631193161011, + 1.720167636871338, + 0.17581266164779663, + 0.42567259073257446, + 0, + 0, + 0.23682871460914612, + 0, + 0, + 0, + 0, + 0, + 1.8280882835388184, + 0.1617840826511383, + 0, + 0.13557660579681396, + 0.5832244157791138, + 0, + 0, + 0.03256487846374512, + 0, + 0, + 0.03892314434051514, + 0, + 0, + 0, + 0.30978846549987793, + 0, + 0, + 0.36915141344070435, + 0, + 0.5477294325828552, + 0, + 0, + 0.06339260935783386, + 0.1851767599582672, + 0.5839155912399292, + 0, + 0, + 0, + 0, + 0, + 0.12337607145309448, + 0, + 0, + 1.0378936529159546, + 0, + 0, + 0, + 0.01616498827934265, + 0.20259439945220947, + 0, + 0, + 0.3087460398674011, + 0.618510365486145, + 0.24435847997665405, + 0, + 0.4668591022491455, + 0.1788468360900879, + 0.200361967086792, + 0, + 0, + 0, + 0, + 0, + 0, + 0.7064645290374756 + ], + [ + 0.2921750843524933, + 0, + 0, + 0.2805737257003784, + 0, + 0, + 0, + 0.3694216012954712, + 0, + 1.1156601905822754, + 1.2807728052139282, + 0, + 0.09175515174865723, + 0, + 0, + 0, + 0.10458803176879883, + 0, + 0.021218180656433105, + 0, + 0, + 0.01699376106262207, + 0.09601330757141113, + 0.054788172245025635, + 0, + 0.030488133430480957, + 0.021512210369110107, + 0.2717320919036865, + 0.29357004165649414, + 0.6420693397521973, + 0.05249035358428955, + 0, + 0.06201601028442383, + 0, + 0.4122554659843445, + 1.821354866027832, + 0.01981794834136963, + 0, + 0.14063221216201782, + 0.05093127489089966, + 0, + 0.32148706912994385, + 0.15257668495178223, + 2.418062686920166, + 0.17348229885101318, + 0.08421656489372253, + 0, + 0.4551248550415039, + 0, + 0.015430927276611328, + 0.24434363842010498, + 0, + 0.06232607364654541, + 0, + 0.04422914981842041, + 0.8720088005065918, + 0.3721686899662018, + 0, + 0, + 0, + 0.340120404958725, + 0, + 0, + 0.07813769578933716, + 0, + 0.0882720947265625, + 0.19706517457962036, + 0.4056885242462158, + 0.19529414176940918, + 0, + 2.928431510925293, + 1.1402223110198975, + 0, + 0.026796698570251465, + 0.0033188462257385254, + 0, + 0.3370524048805237, + 0.47657889127731323, + 0, + 0.10358679294586182, + 0.27619925141334534, + 0, + 0, + 0, + 0.40909066796302795, + 0.2599871754646301, + 0, + 0.275011271238327, + 0.5349323749542236, + 0.07697033882141113, + 0.17431437969207764, + 0, + 0, + 0, + 0.09000074863433838, + 0, + 0, + 0, + 0.276567280292511, + 0, + 0, + 0, + 0, + 0.5655339360237122, + 0, + 0.8971189856529236, + 0, + 0.5199201107025146, + 0, + 0.6301102638244629, + 0.013657361268997192, + 0.04469645023345947, + 0.038062095642089844, + 0.4305816888809204, + 0, + 0.04173767566680908, + 0, + 0, + 0, + 0.8985729217529297, + 0, + 0, + 0, + 0, + 0, + 0.08318889141082764, + 0.006362795829772949, + 2.069222927093506, + 0, + 0.7068352103233337, + 0, + 0.8527798652648926, + 0, + 0, + 0.4707651138305664, + 0, + 0, + 0, + 0.7790955305099487, + 0.021227538585662842, + 0.01846003532409668 + ], + [ + 0, + 0, + 0.2200499176979065, + 0, + 0, + 0, + 0, + 0.2433047890663147, + 0.2504638135433197, + 0.712148904800415, + 0, + 0, + 0, + 0, + 0, + 0.1410943865776062, + 0, + 0, + 0, + 0.11292147636413574, + 0, + 0, + 2.360842704772949, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1.2830760478973389, + 0, + 0, + 0, + 0.6308119893074036, + 0, + 0.4040885865688324, + 0, + 0, + 0, + 0, + 0, + 0.5223236680030823, + 0, + 0, + 0, + 0, + 0.23784160614013672, + 0, + 0.04762387275695801, + 0, + 0, + 0, + 0, + 0.5758676528930664, + 0.01025208830833435, + 0.24556085467338562, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1.1104614734649658, + 1.079118251800537, + 0, + 0, + 0.14462929964065552, + 1.9186956882476807, + 0, + 0.30735498666763306, + 0, + 0, + 0.07669633626937866, + 0, + 0, + 0, + 0, + 0, + 1.3975048065185547, + 0, + 0, + 0.3461639881134033, + 0.5062156915664673, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.19610454142093658, + 0.218009352684021, + 0, + 0, + 0.07953745126724243, + 0, + 0.1416093111038208, + 0, + 0, + 0, + 0.18305465579032898, + 0.10310900211334229, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.45315277576446533, + 0, + 0, + 0, + 0.09076884388923645, + 0, + 0, + 0, + 0, + 0, + 0.04246491193771362, + 0, + 0.1807355284690857, + 0, + 0.3002055883407593, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0.02005404233932495, + 0, + 0, + 0.07601284980773926, + 0, + 0, + 0, + 0.012166053056716919, + 0, + 1.0662918090820312, + 1.4810535907745361, + 0, + 0.014786958694458008, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.1491186022758484, + 0, + 0, + 0, + 0.38226866722106934, + 0.43110355734825134, + 0, + 0, + 0, + 0, + 0, + 1.6819074153900146, + 0, + 0.7939910888671875, + 0.28643298149108887, + 0, + 0, + 0.011532962322235107, + 0, + 1.2869157791137695, + 0, + 0, + 0, + 0, + 0, + 0.16446048021316528, + 0, + 0, + 0, + 0, + 0, + 0, + 0.03375712037086487, + 0, + 0, + 0, + 0.1915181577205658, + 0, + 0, + 0.10225892066955566, + 0, + 0, + 0, + 0.7338485717773438, + 0, + 0, + 1.3715617656707764, + 1.6115869283676147, + 0, + 0.7128411531448364, + 0, + 0, + 0.2161598801612854, + 0.5098914504051208, + 0, + 0, + 0.04084053635597229, + 0, + 0, + 0, + 0.17978456616401672, + 0, + 0, + 0.1365671455860138, + 0.27122950553894043, + 0.2945059537887573, + 0.2824629545211792, + 0, + 0, + 0, + 0.0464092493057251, + 0, + 0, + 0.04672741889953613, + 0.6179839968681335, + 0, + 0, + 0, + 0, + 0.045598745346069336, + 0, + 1.0172381401062012, + 0, + 0.07242608070373535, + 0, + 0.5165215730667114, + 0, + 0, + 0, + 0.5004003047943115, + 0, + 0, + 0, + 0, + 0, + 0.3409433960914612, + 0, + 0.1579979658126831, + 0.09901612997055054, + 0, + 0, + 0, + 0, + 2.413944721221924, + 0, + 0.20971286296844482, + 0.07062971591949463, + 0.26070594787597656, + 0, + 0, + 0, + 0, + 0, + 0.020640969276428223, + 1.0534553527832031, + 0, + 0 + ], + [ + 0, + 0, + 0.046907246112823486, + 0, + 0, + 0, + 0, + 0.20885008573532104, + 0.25957152247428894, + 1.0767037868499756, + 0, + 0, + 0, + 0, + 0, + 0.23976856470108032, + 0, + 0, + 0, + 0, + 0, + 0, + 2.762990951538086, + 0, + 0, + 0, + 0, + 0.29466086626052856, + 0, + 0, + 0.09433537721633911, + 1.2446393966674805, + 0, + 0, + 0, + 0.6668079495429993, + 0, + 0.7482341527938843, + 0, + 0, + 0.005075186491012573, + 0, + 0, + 0.4049275517463684, + 0, + 0, + 0, + 0, + 0.09314888715744019, + 0, + 0, + 0, + 0, + 0, + 0, + 0.4028928279876709, + 0, + 0.3687801659107208, + 0, + 0.10555410385131836, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1.066054105758667, + 1.4596349000930786, + 0, + 0, + 0, + 2.3358588218688965, + 0, + 0.5390753149986267, + 0, + 0, + 0.12931063771247864, + 0, + 0.10619288682937622, + 0, + 0, + 0, + 0.41271400451660156, + 0, + 0, + 0.23865878582000732, + 0.7501264810562134, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.2947666645050049, + 0, + 0, + 0, + 0.05958199501037598, + 0.20450782775878906, + 0, + 0, + 0, + 0.13838836550712585, + 0.13835513591766357, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.45820748805999756, + 0, + 0, + 0, + 0.19962045550346375, + 0, + 0, + 0, + 0, + 0.20416772365570068, + 0.46223968267440796, + 0, + 0.22815394401550293, + 0, + 0.1125795841217041, + 0, + 0, + 0, + 0, + 0, + 0, + 0.3023688793182373 + ], + [ + 0.28365251421928406, + 0, + 0, + 0.41595208644866943, + 0, + 0.15376341342926025, + 0, + 0.22517156600952148, + 0, + 0.7871096134185791, + 1.3084614276885986, + 0.2012956142425537, + 0, + 0, + 0, + 0.2532406449317932, + 0.009012699127197266, + 0, + 0, + 0, + 0, + 0.7235959768295288, + 0.021468758583068848, + 0, + 0, + 0, + 0, + 0.8338297009468079, + 0.3022422790527344, + 0.6702529191970825, + 0.5416026711463928, + 0, + 0, + 0, + 0.2034381628036499, + 1.9052581787109375, + 0, + 0.23752644658088684, + 0, + 0, + 0, + 0.8470145463943481, + 0, + 2.820002555847168, + 0, + 0.16275432705879211, + 0.06714236736297607, + 0.12017238140106201, + 0, + 0, + 0, + 0, + 0.486280620098114, + 0, + 0, + 0.3096342086791992, + 0.3064201772212982, + 0, + 0.09773910045623779, + 0, + 0.4613642394542694, + 0, + 0.021892428398132324, + 0, + 0.18887782096862793, + 0.18538141250610352, + 0, + 0.42975664138793945, + 0.9873132705688477, + 0, + 2.163774013519287, + 1.2928048372268677, + 0, + 0.2320784330368042, + 0.0062233805656433105, + 0, + 1.2478563785552979, + 0.5479208827018738, + 0, + 0.06501156091690063, + 0.3741762936115265, + 0, + 0, + 0.31712013483047485, + 0.5228050947189331, + 0.3981531858444214, + 0, + 0, + 0.4854400157928467, + 0.3341655731201172, + 0.39207732677459717, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.3316766023635864, + 0, + 0, + 0.33435362577438354, + 0.1380615234375, + 0.7183249592781067, + 0.041296958923339844, + 0.7634149193763733, + 0, + 0.4028007984161377, + 0, + 0.6915435791015625, + 0, + 0, + 0, + 0.3831353187561035, + 0.05798754096031189, + 0.15244710445404053, + 0, + 0.03230410814285278, + 0.2039397656917572, + 0.6142292022705078, + 0.15542924404144287, + 0.07628917694091797, + 0.0812273919582367, + 0.15177401900291443, + 0.10224854946136475, + 0, + 0, + 2.8106069564819336, + 0.3994237184524536, + 0.6397127509117126, + 0, + 0.8949670791625977, + 0, + 0, + 0.18832790851593018, + 0.1450880765914917, + 0, + 0, + 0.6900937557220459, + 0, + 0.14745783805847168 + ], + [ + 0.12055802345275879, + 0.023864269256591797, + 0, + 0, + 0, + 0, + 0, + 0.3327372670173645, + 0.1789897382259369, + 1.1445300579071045, + 0, + 0, + 0, + 0, + 0, + 0.4361664652824402, + 0.09996795654296875, + 0.10051405429840088, + 0, + 0.4030296802520752, + 0.06672021746635437, + 0.6339577436447144, + 3.3947582244873047, + 0, + 0, + 0, + 0, + 0.9711236357688904, + 0, + 0.38066884875297546, + 0.4158353805541992, + 1.5344438552856445, + 0, + 0.19816407561302185, + 0, + 0.6646860241889954, + 0, + 0.16733816266059875, + 0, + 0, + 0, + 0.322623074054718, + 0, + 0.7314171195030212, + 0, + 0, + 0, + 0, + 0.043955981731414795, + 0, + 0, + 0, + 0, + 0, + 0, + 0.9436180591583252, + 0, + 0.29259607195854187, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.1570979356765747, + 0, + 0, + 0, + 1.1782727241516113, + 1.2431498765945435, + 0.32878363132476807, + 0, + 0.419150173664093, + 2.3304405212402344, + 0.8566346764564514, + 0, + 0, + 0, + 0.3841046392917633, + 0.10476112365722656, + 0, + 0.18140661716461182, + 0, + 0, + 0.6665420532226562, + 0, + 0, + 0.22877633571624756, + 0.9225524663925171, + 0, + 0.15886402130126953, + 0, + 0, + 0.02094721794128418, + 0, + 0, + 0, + 0.3046541213989258, + 0.2845715284347534, + 0, + 0, + 0.4244043231010437, + 0.164473295211792, + 0.30073386430740356, + 0.7123112678527832, + 0.1730642318725586, + 0, + 0.4041661322116852, + 0.39166414737701416, + 0, + 0, + 0.2103893756866455, + 0.007811635732650757, + 0.010994672775268555, + 0.03914850950241089, + 0, + 0, + 0.8430832624435425, + 0, + 0, + 0, + 0.15830591320991516, + 0.29398930072784424, + 0, + 0, + 0, + 0.5994948148727417, + 0.1704254150390625, + 0, + 0.4673898220062256, + 0, + 0.3204514980316162, + 0, + 0, + 0, + 0, + 0, + 0, + 0.8447363376617432 + ] + ] + } + ], + "layout": { + "coloraxis": { + "cmid": 0, + "colorscale": [ + [ + 0, + "rgb(103,0,31)" + ], + [ + 0.1, + "rgb(178,24,43)" + ], + [ + 0.2, + "rgb(214,96,77)" + ], + [ + 0.3, + "rgb(244,165,130)" + ], + [ + 0.4, + "rgb(253,219,199)" + ], + [ + 0.5, + "rgb(247,247,247)" + ], + [ + 0.6, + "rgb(209,229,240)" + ], + [ + 0.7, + "rgb(146,197,222)" + ], + [ + 0.8, + "rgb(67,147,195)" + ], + [ + 0.9, + "rgb(33,102,172)" + ], + [ + 1, + "rgb(5,48,97)" + ] + ] + }, + "template": { + "data": { + "bar": [ + { + "error_x": { + "color": "#2a3f5f" + }, + "error_y": { + "color": "#2a3f5f" + }, + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "bar" + } + ], + "barpolar": [ + { + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "barpolar" + } + ], + "carpet": [ + { + "aaxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "baxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "type": "carpet" + } + ], + "choropleth": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "choropleth" + } + ], + "contour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "contour" + } + ], + "contourcarpet": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "contourcarpet" + } + ], + "heatmap": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmap" + } + ], + "heatmapgl": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmapgl" + } + ], + "histogram": [ + { + "marker": { + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "histogram" + } + ], + "histogram2d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2d" + } + ], + "histogram2dcontour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2dcontour" + } + ], + "mesh3d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "mesh3d" + } + ], + "parcoords": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "parcoords" + } + ], + "pie": [ + { + "automargin": true, + "type": "pie" + } + ], + "scatter": [ + { + "fillpattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + }, + "type": "scatter" + } + ], + "scatter3d": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatter3d" + } + ], + "scattercarpet": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattercarpet" + } + ], + "scattergeo": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergeo" + } + ], + "scattergl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergl" + } + ], + "scattermapbox": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermapbox" + } + ], + "scatterpolar": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolar" + } + ], + "scatterpolargl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolargl" + } + ], + "scatterternary": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterternary" + } + ], + "surface": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "surface" + } + ], + "table": [ + { + "cells": { + "fill": { + "color": "#EBF0F8" + }, + "line": { + "color": "white" + } + }, + "header": { + "fill": { + "color": "#C8D4E3" + }, + "line": { + "color": "white" + } + }, + "type": "table" + } + ] + }, + "layout": { + "annotationdefaults": { + "arrowcolor": "#2a3f5f", + "arrowhead": 0, + "arrowwidth": 1 + }, + "autotypenumbers": "strict", + "coloraxis": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "colorscale": { + "diverging": [ + [ + 0, + "#8e0152" + ], + [ + 0.1, + "#c51b7d" + ], + [ + 0.2, + "#de77ae" + ], + [ + 0.3, + "#f1b6da" + ], + [ + 0.4, + "#fde0ef" + ], + [ + 0.5, + "#f7f7f7" + ], + [ + 0.6, + "#e6f5d0" + ], + [ + 0.7, + "#b8e186" + ], + [ + 0.8, + "#7fbc41" + ], + [ + 0.9, + "#4d9221" + ], + [ + 1, + "#276419" + ] + ], + "sequential": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "sequentialminus": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ] + }, + "colorway": [ + "#636efa", + "#EF553B", + "#00cc96", + "#ab63fa", + "#FFA15A", + "#19d3f3", + "#FF6692", + "#B6E880", + "#FF97FF", + "#FECB52" + ], + "font": { + "color": "#2a3f5f" + }, + "geo": { + "bgcolor": "white", + "lakecolor": "white", + "landcolor": "#E5ECF6", + "showlakes": true, + "showland": true, + "subunitcolor": "white" + }, + "hoverlabel": { + "align": "left" + }, + "hovermode": "closest", + "mapbox": { + "style": "light" + }, + "paper_bgcolor": "white", + "plot_bgcolor": "#E5ECF6", + "polar": { + "angularaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "radialaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "scene": { + "xaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "yaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "zaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + } + }, + "shapedefaults": { + "line": { + "color": "#2a3f5f" + } + }, + "ternary": { + "aaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "baxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "caxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "title": { + "x": 0.05 + }, + "xaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + }, + "yaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + } + } + }, + "title": { + "text": "Activations of Live SAE features at L5 S2 position per prompt" + }, + "xaxis": { + "anchor": "y", + "constrain": "domain", + "domain": [ + 0, + 1 + ], + "scaleanchor": "y", + "title": { + "text": "Feature Id" + } + }, + "yaxis": { + "anchor": "x", + "autorange": "reversed", + "constrain": "domain", + "domain": [ + 0, + 1 + ], + "title": { + "text": "Prompt" + } + } + } + }, + "text/html": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "layer, s2_pos = 5, 10\n", + "saes = [hook_name_to_sae[utils.get_act_name('z', layer)]]\n", + "_, cache = model.run_with_cache_with_saes(tokens, saes=saes)\n", + "sae_acts = cache[utils.get_act_name('z', layer) + \".hook_sae_acts_post\"][:, s2_pos, :]\n", + "live_feature_mask = sae_acts > 0\n", + "live_feature_union = live_feature_mask.any(dim=0)\n", + "\n", + "imshow(\n", + " sae_acts[:, live_feature_union],\n", + " title = \"Activations of Live SAE features at L5 S2 position per prompt\",\n", + " xaxis=\"Feature Id\", yaxis=\"Prompt\",\n", + " x=list(map(str, live_feature_union.nonzero().flatten().tolist())),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We could then interpret some of the commonly activating features, like 7515, using [neuronpedia](https://www.neuronpedia.org/gpt2-small/5-att-kk/7515)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Run with Hooks (with SAEs)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "HookedSAETransformer also allows you to intervene on SAE activations with `model.run_with_hooks_with_saes(tokens, saes=saes, fwd_hooks=fwd_hooks)`. This works exactly like the standard TransformerLens `run_with_hooks`, with the added benefit that we can now intervene on SAE activations from the HookedSAEs that we splice in. Along the same lines as `run_with_saes` and `run_with_cache_with_saes`, this will only temporarily add SAEs before returning the model to it's original state. \n", + "\n", + "I expect this to be useful when doing circuit analysis with SAEs. To demonstrate, let's zero ablate individual layer 5 attention SAE features to localize causally important features." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 141/141 [00:04<00:00, 28.85it/s]\n" + ] + }, + { + "data": { + "application/vnd.plotly.v1+json": { + "config": { + "plotlyServerURL": "https://plot.ly" + }, + "data": [ + { + "coloraxis": "coloraxis", + "hovertemplate": "Feature Idx: %{x}
Prompt Idx: %{y}
color: %{z}", + "name": "0", + "type": "heatmap", + "x": [ + "46", + "345", + "702", + "1372", + "1755", + "1965", + "2457", + "2496", + "2646", + "2999", + "3047", + "4569", + "5132", + "5203", + "5508", + "5940", + "6144", + "6371", + "6515", + "6558", + "6812", + "7092", + "7515", + "7907", + "8063", + "8623", + "8737", + "8768", + "9096", + "9102", + "9186", + "9463", + "9746", + "9913", + "10581", + "10894", + "12109", + "12485", + "12764", + "12866", + "13063", + "13624", + "13707", + "13777", + "14844", + "15050", + "15170", + "15696", + "16178", + "16892", + "17156", + "17259", + "17497", + "17854", + "18043", + "18210", + "18318", + "18385", + "18440", + "18920", + "19183", + "19263", + "19442", + "19524", + "19573", + "20838", + "21151", + "21657", + "22108", + "23578", + "24091", + "24217", + "25792", + "26373", + "26410", + "27535", + "27787", + "27811", + "27960", + "28061", + "28241", + "28242", + "28254", + "28349", + "28977", + "29027", + "29482", + "29603", + "29700", + "29822", + "32177", + "32920", + "33320", + "33730", + "33966", + "34177", + "34334", + "34947", + "35403", + "35425", + "35579", + "35665", + "35815", + "36109", + "36172", + "36451", + "36767", + "36917", + "38570", + "39962", + "40409", + "40418", + "40661", + "41162", + "41185", + "41552", + "42024", + "42161", + "42437", + "42577", + "42882", + "42931", + "43035", + "43414", + "43643", + "43662", + "44203", + "44256", + "44452", + "44652", + "45179", + "45814", + "45984", + "46880", + "47117", + "47170", + "47231", + "47313", + "47680", + "48063", + "48703" + ], + "xaxis": "x", + "yaxis": "y", + "z": [ + [ + 0.006268501281738281, + 0, + 0, + 0.0016260147094726562, + 0.0011568069458007812, + 0, + 0, + -0.000400543212890625, + 0, + -0.024961471557617188, + -0.062079429626464844, + 0, + 0.00041866302490234375, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + -0.017510414123535156, + -0.0021581649780273438, + -0.0012054443359375, + -0.006356239318847656, + 0, + 0, + 0, + 0, + 0.025524139404296875, + 0, + 0, + -0.0037746429443359375, + 0.0004291534423828125, + -0.000194549560546875, + 0.002796173095703125, + 0.0001850128173828125, + -0.056549072265625, + -0.0029163360595703125, + -0.004790306091308594, + 0, + 0.0005321502685546875, + 0, + 0.00049591064453125, + -0.0008335113525390625, + 0, + -0.00299072265625, + -0.00185394287109375, + 0, + 0, + 0.011702537536621094, + 0, + 0, + 0, + -0.003353118896484375, + 0, + 0, + 0, + 0.00048828125, + -0.000213623046875, + 0, + -0.0062160491943359375, + -0.007611274719238281, + 0, + 0.06644821166992188, + -0.025884628295898438, + 0, + -0.0001964569091796875, + 0, + 0, + 0.03233909606933594, + -0.05103874206542969, + 0.0003414154052734375, + -0.0000057220458984375, + -0.0027713775634765625, + 0, + 0, + 0, + -0.02438068389892578, + 0.027306556701660156, + 0, + -0.0036411285400390625, + 0.018335342407226562, + 0.010270118713378906, + 0.0120849609375, + 0.0013589859008789062, + 0, + 0, + -0.0033817291259765625, + 0, + 0, + 0, + -0.014057159423828125, + 0, + 0, + 0, + 0, + -0.008485794067382812, + 0, + 0.021463394165039062, + 0, + -0.002582550048828125, + 0, + 0.012966156005859375, + 0, + 0, + 0, + -0.0077991485595703125, + 0.002948760986328125, + 0.0069675445556640625, + 0, + 0, + 0.0058879852294921875, + -0.050632476806640625, + 0.001888275146484375, + 0, + 0, + -0.0005016326904296875, + 0, + 0, + 0, + -0.5087032318115234, + -0.0006818771362304688, + 0.0017566680908203125, + 0, + -0.02089214324951172, + -0.0000286102294921875, + 0, + 0, + -0.000446319580078125, + 0.0008115768432617188, + 0, + 0.017795562744140625, + 0, + -0.008462905883789062 + ], + [ + 0, + 0, + 0.0042266845703125, + 0, + 0, + 0, + -0.00130462646484375, + -0.01946258544921875, + 0, + 0.03999900817871094, + 0.013164520263671875, + 0, + 0, + -0.000522613525390625, + -0.0028820037841796875, + -0.003643035888671875, + 0, + 0, + 0, + 0, + 0, + 0, + -0.24383163452148438, + 0, + 0, + -0.0009517669677734375, + 0, + 0.05923271179199219, + 0.00897979736328125, + 0, + 0, + -0.00617218017578125, + 0, + 0.011938095092773438, + 0.005764007568359375, + 0.08927345275878906, + 0, + 0, + 0, + 0, + 0.027820587158203125, + 0, + 0, + 0.021488189697265625, + 0, + 0, + 0, + 0, + 0.016414642333984375, + 0, + -0.012666702270507812, + 0.002353668212890625, + 0, + 0, + 0, + 0.10541152954101562, + 0, + 0.010334014892578125, + 0, + 0, + 0, + 0.0012111663818359375, + 0, + 0, + 0, + 0, + -0.047576904296875, + 0, + 0, + -0.006137847900390625, + 0.04940223693847656, + 0.014007568359375, + 0.030317306518554688, + 0, + -0.0012969970703125, + -0.12521743774414062, + 0.0023975372314453125, + 0.04903602600097656, + 0, + 0, + 0.019681930541992188, + 0, + 0, + 0, + 0, + 0, + -0.07957077026367188, + -0.00966644287109375, + 0, + 0.011016845703125, + 0.05775642395019531, + 0, + 0, + 0.00060272216796875, + 0, + 0, + 0.00067138671875, + 0, + 0, + 0, + -0.0041980743408203125, + 0, + 0, + 0.020341873168945312, + 0, + -0.02782440185546875, + 0, + 0, + 0.001705169677734375, + 0.0035266876220703125, + 0.0060558319091796875, + 0, + 0, + 0, + 0, + 0, + 0.0004119873046875, + 0, + 0, + 0.10181617736816406, + 0, + 0, + 0, + 0.0001964569091796875, + 0.009687423706054688, + 0, + 0, + 0.10214805603027344, + 0.03883934020996094, + 0.028743743896484375, + 0, + -0.009389877319335938, + -0.0005168914794921875, + -0.0241851806640625, + 0, + 0, + 0, + 0, + 0, + 0, + 0.0089263916015625 + ], + [ + 0.013156890869140625, + 0, + 0, + 0.00737762451171875, + 0, + 0, + 0, + -0.011926651000976562, + 0, + -0.1016092300415039, + -0.2541160583496094, + 0, + 0.0026063919067382812, + 0, + 0, + 0, + 0.011356353759765625, + 0, + -0.0003261566162109375, + 0, + 0, + 0.000354766845703125, + 0.018985748291015625, + -0.0010251998901367188, + 0, + -0.0016918182373046875, + 0.00087738037109375, + -0.03418159484863281, + -0.022599220275878906, + -0.031129837036132812, + -0.0039033889770507812, + 0, + 0.002773284912109375, + 0, + -0.0497589111328125, + 0.0000972747802734375, + 0.00002002716064453125, + 0, + -0.000766754150390625, + 0.000133514404296875, + 0, + 0.00109100341796875, + 0.00045013427734375, + -0.15281009674072266, + -0.0027723312377929688, + -0.008421897888183594, + 0, + 0.024028778076171875, + 0, + 0.0008792877197265625, + -0.0008392333984375, + 0, + -0.014632225036621094, + 0, + -0.0009860992431640625, + -0.0236358642578125, + 0.021772384643554688, + 0, + 0, + 0, + -0.016798019409179688, + 0, + 0, + -0.0022678375244140625, + 0, + -0.0038995742797851562, + 0.006114959716796875, + -0.05572509765625, + -0.008089065551757812, + 0, + 0.21244430541992188, + -0.06043434143066406, + 0, + 0.0001010894775390625, + 0.00023651123046875, + 0, + 0.062018394470214844, + -0.08936023712158203, + 0, + -0.005387306213378906, + -0.001903533935546875, + 0, + 0, + 0, + -0.08661651611328125, + 0.020143508911132812, + 0, + -0.01000213623046875, + 0.008556365966796875, + -0.0023040771484375, + 0.0063114166259765625, + 0, + 0, + 0, + -0.01030731201171875, + 0, + 0, + 0, + -0.037540435791015625, + 0, + 0, + 0, + 0, + -0.018768310546875, + 0, + 0.06715202331542969, + 0, + -0.01861572265625, + 0, + 0.02222919464111328, + -0.0029458999633789062, + -0.0005445480346679688, + -0.001338958740234375, + -0.0246734619140625, + 0, + 0.0014019012451171875, + 0, + 0, + 0, + -0.34259986877441406, + 0, + 0, + 0, + 0, + 0, + -0.002704620361328125, + -0.0001850128173828125, + -0.9704685211181641, + 0, + -0.01996612548828125, + 0, + -0.0199432373046875, + 0, + 0, + 0.025028228759765625, + 0, + 0, + 0, + 0.05844879150390625, + -0.00006961822509765625, + -0.002410888671875 + ], + [ + 0, + 0, + -0.001018524169921875, + 0, + 0, + 0, + 0, + -0.0172882080078125, + 0.05738639831542969, + 0.12810707092285156, + 0, + 0, + 0, + 0, + 0, + -0.0056362152099609375, + 0, + 0, + 0, + 0.009425163269042969, + 0, + 0, + -0.2314128875732422, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + -0.057198524475097656, + 0, + 0, + 0, + 0.13471412658691406, + 0, + 0.08182525634765625, + 0, + 0, + 0, + 0, + 0, + 0.006465911865234375, + 0, + 0, + 0, + 0, + 0.0039052963256835938, + 0, + -0.0010318756103515625, + 0, + 0, + 0, + 0, + 0.062198638916015625, + 0.0000057220458984375, + -0.001708984375, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.03947257995605469, + 0.1576099395751953, + 0, + 0, + 0.00009822845458984375, + -0.25530242919921875, + 0, + 0.061611175537109375, + 0, + 0, + 0.0061016082763671875, + 0, + 0, + 0, + 0, + 0, + -0.079315185546875, + 0, + 0, + 0.04389762878417969, + 0.06207084655761719, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.0064945220947265625, + -0.009065628051757812, + 0, + 0, + 0.0025882720947265625, + 0, + 0.0033740997314453125, + 0, + 0, + 0, + 0.014276504516601562, + -0.011219978332519531, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.023397445678710938, + 0, + 0, + 0, + 0.0096435546875, + 0, + 0, + 0, + 0, + 0, + 0.007327079772949219, + 0, + 0.00238037109375, + 0, + -0.04556846618652344, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + -0.0007219314575195312, + 0, + 0, + -0.001102447509765625, + 0, + 0, + 0, + -0.00047397613525390625, + 0, + -0.02031421661376953, + -0.18840694427490234, + 0, + 0.0009374618530273438, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + -0.0014810562133789062, + 0, + 0, + 0, + -0.01897907257080078, + -0.012393951416015625, + 0, + 0, + 0, + 0, + 0, + -0.007961273193359375, + 0, + 0.006266593933105469, + 0.022070884704589844, + 0, + 0, + -0.00022220611572265625, + 0, + -0.08554744720458984, + 0, + 0, + 0, + 0, + 0, + 0.00211334228515625, + 0, + 0, + 0, + 0, + 0, + 0, + -0.0006618499755859375, + 0, + 0, + 0, + 0.00042629241943359375, + 0, + 0, + -0.0023794174194335938, + 0, + 0, + 0, + -0.08295249938964844, + 0, + 0, + 0.02340221405029297, + 0.05393028259277344, + 0, + 0.0030164718627929688, + 0, + 0, + 0.02137470245361328, + -0.0648040771484375, + 0, + 0, + -0.0007104873657226562, + 0, + 0, + 0, + -0.02891063690185547, + 0, + 0, + -0.0024862289428710938, + -0.007077217102050781, + -0.004982948303222656, + 0.004157066345214844, + 0, + 0, + 0, + -0.0009584426879882812, + 0, + 0, + -0.0016260147094726562, + -0.03653144836425781, + 0, + 0, + 0, + 0, + -0.004261970520019531, + 0, + 0.1517467498779297, + 0, + -0.0017957687377929688, + 0, + 0.01949596405029297, + 0, + 0, + 0, + -0.024643898010253906, + 0, + 0, + 0, + 0, + 0, + -0.12193775177001953, + 0, + 0.01824474334716797, + 0.006918907165527344, + 0, + 0, + 0, + 0, + -0.5964584350585938, + 0, + -0.004886627197265625, + -0.0028219223022460938, + -0.013730049133300781, + 0, + 0, + 0, + 0, + 0, + 0.000370025634765625, + 0.11502552032470703, + 0, + 0 + ], + [ + 0, + 0, + 0.0020799636840820312, + 0, + 0, + 0, + 0, + -0.02874469757080078, + 0.0672769546508789, + 0.31006431579589844, + 0, + 0, + 0, + 0, + 0, + -0.014065742492675781, + 0, + 0, + 0, + 0, + 0, + 0, + -0.42875194549560547, + 0, + 0, + 0, + 0, + 0.037166595458984375, + 0, + 0, + 0.00395965576171875, + -0.09044742584228516, + 0, + 0, + 0, + 0.16284751892089844, + 0, + 0.2745513916015625, + 0, + 0, + 0.0013599395751953125, + 0, + 0, + -0.016633033752441406, + 0, + 0, + 0, + 0, + 0.002765655517578125, + 0, + 0, + 0, + 0, + 0, + 0, + 0.06857013702392578, + 0, + 0.0030755996704101562, + 0, + 0.005713462829589844, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.010555267333984375, + 0.35628509521484375, + 0, + 0, + 0, + -0.3705453872680664, + 0, + 0.1321268081665039, + 0, + 0, + 0.01171875, + 0, + 0.006653785705566406, + 0, + 0, + 0, + -0.04768085479736328, + 0, + 0, + 0.05365467071533203, + 0.10848140716552734, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + -0.0019636154174804688, + 0, + 0, + 0, + -0.0038604736328125, + -0.00696563720703125, + 0, + 0, + 0, + 0.004207611083984375, + -0.009866714477539062, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.041828155517578125, + 0, + 0, + 0, + 0.03432941436767578, + 0, + 0, + 0, + 0, + 0.02262592315673828, + 0.1012563705444336, + 0, + 0.0032415390014648438, + 0, + -0.028539657592773438, + 0, + 0, + 0, + 0, + 0, + 0, + 0.019530296325683594 + ], + [ + 0.0072574615478515625, + 0, + 0, + 0.0045604705810546875, + 0, + -0.002410888671875, + 0, + 0.000942230224609375, + 0, + -0.028242111206054688, + -0.06697559356689453, + -0.002197265625, + 0, + 0, + 0, + 0.01448822021484375, + 0.00038909912109375, + 0, + 0, + 0, + 0, + -0.0072345733642578125, + 0.0015048980712890625, + 0, + 0, + 0, + 0, + -0.026609420776367188, + -0.007898330688476562, + 0.006641387939453125, + -0.012470245361328125, + 0, + 0, + 0, + -0.0054531097412109375, + 0.06533622741699219, + 0, + 0.00041484832763671875, + 0, + 0, + 0, + -0.002368927001953125, + 0, + 0.04226112365722656, + 0, + -0.0031299591064453125, + -0.0000457763671875, + 0.000308990478515625, + 0, + 0, + 0, + 0, + -0.0275726318359375, + 0, + 0, + -0.004794120788574219, + 0.01718902587890625, + 0, + -0.001049041748046875, + 0, + -0.007875442504882812, + 0, + -0.00032806396484375, + 0, + 0.002880096435546875, + -0.0073566436767578125, + 0, + -0.012141227722167969, + -0.002796173095703125, + 0, + 0.0904073715209961, + -0.020002365112304688, + 0, + 0.0006046295166015625, + 0.0000095367431640625, + 0, + 0.09020233154296875, + -0.024329185485839844, + 0, + -0.0007257461547851562, + 0.0022792816162109375, + 0, + 0, + 0.0024671554565429688, + -0.031095504760742188, + 0.029073715209960938, + 0, + 0, + 0.017263412475585938, + 0.009774208068847656, + 0.01905059814453125, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + -0.007511138916015625, + 0, + 0, + -0.01740264892578125, + -0.012363433837890625, + -0.007237434387207031, + 0.00046825408935546875, + 0.015039443969726562, + 0, + -0.001247406005859375, + 0, + 0.04442596435546875, + 0, + 0, + 0, + 0.0020885467529296875, + 0.0009975433349609375, + 0.0068645477294921875, + 0, + 0.0009918212890625, + 0.007763862609863281, + -0.10830020904541016, + 0.002170562744140625, + 0.0041522979736328125, + 0.0009832382202148438, + -0.0055789947509765625, + -0.0020475387573242188, + 0, + 0, + -0.46219825744628906, + -0.0004138946533203125, + 0.022248268127441406, + 0, + -0.023275375366210938, + 0, + 0, + -0.00007152557373046875, + -0.0017099380493164062, + 0, + 0, + 0.028047561645507812, + 0, + -0.006505012512207031 + ], + [ + 0.0026121139526367188, + 0.0023622512817382812, + 0, + 0, + 0, + 0, + 0, + -0.04861927032470703, + 0.04393959045410156, + 0.24942588806152344, + 0, + 0, + 0, + 0, + 0, + -0.0894918441772461, + 0.011738777160644531, + 0.0023365020751953125, + 0, + 0.03142070770263672, + 0.007035255432128906, + 0.013895988464355469, + -0.38878440856933594, + 0, + 0, + 0, + 0, + 0.3524456024169922, + 0, + 0.04943275451660156, + 0.07975196838378906, + -0.13926124572753906, + 0, + 0.007584571838378906, + 0, + 0.10158729553222656, + 0, + 0.048768043518066406, + 0, + 0, + 0, + -0.010777473449707031, + 0, + -0.02371692657470703, + 0, + 0, + 0, + 0, + -0.0021333694458007812, + 0, + 0, + 0, + 0, + 0, + 0, + 0.14519309997558594, + 0, + -0.023756027221679688, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + -0.038219451904296875, + 0, + 0, + 0, + -0.07305049896240234, + 0.1724720001220703, + 0.035521507263183594, + 0, + 0.026566505432128906, + -0.2165508270263672, + -0.010828971862792969, + 0, + 0, + 0, + 0.06682586669921875, + 0.0020055770874023438, + 0, + 0.05693340301513672, + 0, + 0, + -0.1571969985961914, + 0, + 0, + 0.0275726318359375, + 0.09813213348388672, + 0, + -0.0074253082275390625, + 0, + 0, + -0.00006008148193359375, + 0, + 0, + 0, + 0.007464408874511719, + -0.011278152465820312, + 0, + 0, + 0.008585929870605469, + -0.02161121368408203, + -0.05259227752685547, + 0.15187358856201172, + 0.009034156799316406, + 0, + 0.01724529266357422, + 0.02186107635498047, + 0, + 0, + 0.023595809936523438, + 0.0018739700317382812, + 0.0014142990112304688, + 0.0001888275146484375, + 0, + 0, + 0.14745807647705078, + 0, + 0, + 0, + 0.022150039672851562, + 0.04754352569580078, + 0, + 0, + 0, + 0.12122058868408203, + 0.037743568420410156, + 0, + -0.022559165954589844, + 0, + -0.07815361022949219, + 0, + 0, + 0, + 0, + 0, + 0, + 0.1304798126220703 + ] + ] + } + ], + "layout": { + "coloraxis": { + "cmid": 0, + "colorscale": [ + [ + 0, + "rgb(103,0,31)" + ], + [ + 0.1, + "rgb(178,24,43)" + ], + [ + 0.2, + "rgb(214,96,77)" + ], + [ + 0.3, + "rgb(244,165,130)" + ], + [ + 0.4, + "rgb(253,219,199)" + ], + [ + 0.5, + "rgb(247,247,247)" + ], + [ + 0.6, + "rgb(209,229,240)" + ], + [ + 0.7, + "rgb(146,197,222)" + ], + [ + 0.8, + "rgb(67,147,195)" + ], + [ + 0.9, + "rgb(33,102,172)" + ], + [ + 1, + "rgb(5,48,97)" + ] + ] + }, + "template": { + "data": { + "bar": [ + { + "error_x": { + "color": "#2a3f5f" + }, + "error_y": { + "color": "#2a3f5f" + }, + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "bar" + } + ], + "barpolar": [ + { + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "barpolar" + } + ], + "carpet": [ + { + "aaxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "baxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "type": "carpet" + } + ], + "choropleth": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "choropleth" + } + ], + "contour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "contour" + } + ], + "contourcarpet": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "contourcarpet" + } + ], + "heatmap": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmap" + } + ], + "heatmapgl": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmapgl" + } + ], + "histogram": [ + { + "marker": { + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "histogram" + } + ], + "histogram2d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2d" + } + ], + "histogram2dcontour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2dcontour" + } + ], + "mesh3d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "mesh3d" + } + ], + "parcoords": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "parcoords" + } + ], + "pie": [ + { + "automargin": true, + "type": "pie" + } + ], + "scatter": [ + { + "fillpattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + }, + "type": "scatter" + } + ], + "scatter3d": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatter3d" + } + ], + "scattercarpet": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattercarpet" + } + ], + "scattergeo": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergeo" + } + ], + "scattergl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergl" + } + ], + "scattermapbox": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermapbox" + } + ], + "scatterpolar": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolar" + } + ], + "scatterpolargl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolargl" + } + ], + "scatterternary": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterternary" + } + ], + "surface": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "surface" + } + ], + "table": [ + { + "cells": { + "fill": { + "color": "#EBF0F8" + }, + "line": { + "color": "white" + } + }, + "header": { + "fill": { + "color": "#C8D4E3" + }, + "line": { + "color": "white" + } + }, + "type": "table" + } + ] + }, + "layout": { + "annotationdefaults": { + "arrowcolor": "#2a3f5f", + "arrowhead": 0, + "arrowwidth": 1 + }, + "autotypenumbers": "strict", + "coloraxis": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "colorscale": { + "diverging": [ + [ + 0, + "#8e0152" + ], + [ + 0.1, + "#c51b7d" + ], + [ + 0.2, + "#de77ae" + ], + [ + 0.3, + "#f1b6da" + ], + [ + 0.4, + "#fde0ef" + ], + [ + 0.5, + "#f7f7f7" + ], + [ + 0.6, + "#e6f5d0" + ], + [ + 0.7, + "#b8e186" + ], + [ + 0.8, + "#7fbc41" + ], + [ + 0.9, + "#4d9221" + ], + [ + 1, + "#276419" + ] + ], + "sequential": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "sequentialminus": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ] + }, + "colorway": [ + "#636efa", + "#EF553B", + "#00cc96", + "#ab63fa", + "#FFA15A", + "#19d3f3", + "#FF6692", + "#B6E880", + "#FF97FF", + "#FECB52" + ], + "font": { + "color": "#2a3f5f" + }, + "geo": { + "bgcolor": "white", + "lakecolor": "white", + "landcolor": "#E5ECF6", + "showlakes": true, + "showland": true, + "subunitcolor": "white" + }, + "hoverlabel": { + "align": "left" + }, + "hovermode": "closest", + "mapbox": { + "style": "light" + }, + "paper_bgcolor": "white", + "plot_bgcolor": "#E5ECF6", + "polar": { + "angularaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "radialaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "scene": { + "xaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "yaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "zaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + } + }, + "shapedefaults": { + "line": { + "color": "#2a3f5f" + } + }, + "ternary": { + "aaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "baxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "caxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "title": { + "x": 0.05 + }, + "xaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + }, + "yaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + } + } + }, + "title": { + "text": "Change in logit diff when ablating L5 SAE features for all prompts at pos 10" + }, + "xaxis": { + "anchor": "y", + "constrain": "domain", + "domain": [ + 0, + 1 + ], + "scaleanchor": "y", + "title": { + "text": "Feature Idx" + } + }, + "yaxis": { + "anchor": "x", + "autorange": "reversed", + "constrain": "domain", + "domain": [ + 0, + 1 + ], + "title": { + "text": "Prompt Idx" + } + } + } + }, + "text/html": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def ablate_sae_feature(sae_acts, hook, pos, feature_id):\n", + " if pos is None:\n", + " sae_acts[:, :, feature_id] = 0.\n", + " else:\n", + " sae_acts[:, pos, feature_id] = 0.\n", + " return sae_acts\n", + "\n", + "layer = 5\n", + "sae = hook_name_to_sae[utils.get_act_name('z', layer)]\n", + "\n", + "logits_with_saes = model.run_with_saes(tokens, saes=sae)\n", + "clean_sae_baseline_per_prompt = logits_to_ave_logit_diff(logits_with_saes, answer_tokens, per_prompt=True)\n", + "\n", + "all_live_features = torch.arange(sae.cfg.d_sae)[live_feature_union.cpu()]\n", + "\n", + "causal_effects = torch.zeros((len(prompts), all_live_features.shape[0]))\n", + "fid_to_idx = {fid.item(): idx for idx, fid in enumerate(all_live_features)}\n", + "\n", + "\n", + "abl_layer, abl_pos = 5, 10\n", + "for feature_id in tqdm.tqdm(all_live_features):\n", + " feature_id = feature_id.item()\n", + " abl_feature_logits = model.run_with_hooks_with_saes(\n", + " tokens,\n", + " saes=sae,\n", + " fwd_hooks=[(utils.get_act_name('z', abl_layer) + \".hook_sae_acts_post\", partial(ablate_sae_feature, pos=abl_pos, feature_id=feature_id))]\n", + " ) # [batch, seq, vocab]\n", + " \n", + " abl_feature_logit_diff = logits_to_ave_logit_diff(abl_feature_logits, answer_tokens, per_prompt=True) # [batch]\n", + " causal_effects[:, fid_to_idx[feature_id]] = abl_feature_logit_diff - clean_sae_baseline_per_prompt\n", + "\n", + "\n", + "imshow(causal_effects, title=f\"Change in logit diff when ablating L{abl_layer} SAE features for all prompts at pos {abl_pos}\", xaxis=\"Feature Idx\", yaxis=\"Prompt Idx\", x=list(map(str, all_live_features.tolist())))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Although it's not super clean, we see a few features stand out, where ablating them causes a nontrivial drop in logit diff on multiple prompts: 7515 and 27535 for BABA prompts, with 44256 for ABBA prompts." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Add SAEs" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "While the `run_with_saes` family of methods are great for evaluating SAEs and exploratory analysis, you may want to permanently attach SAEs to your model. You can attach SAEs to any activation with `model.add_sae(sae)`, where sae is a HookedSAE. \n", + "\n", + "When you add an SAE, it gets stored in `model.acts_to_saes`, a dictionary that maps the activation name to the HookedSAE that is attached. The main benefit of permanently adding SAEs is that we can now just run the model like a normal HookedTransformer (with `forward`, `run_with_cache`, `run_with_hooks`), but some activations will be replaced with the reconstructed activations from the corresponding SAEs.\n", + "\n", + "I expect this to be most useful when you've already identified a good set of SAEs that you want to use for interpretability, and don't feel like passing in a massive list of saes for every forward pass." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Attached SAEs before add_sae {}\n", + "Attached SAEs after add_sae {'blocks.5.attn.hook_z': HookedSAE(\n", + " (hook_sae_input): HookPoint()\n", + " (hook_sae_acts_pre): HookPoint()\n", + " (hook_sae_acts_post): HookPoint()\n", + " (hook_sae_recons): HookPoint()\n", + " (hook_sae_error): HookPoint()\n", + " (hook_sae_output): HookPoint()\n", + ")}\n" + ] + } + ], + "source": [ + "print(\"Attached SAEs before add_sae\", model.acts_to_saes)\n", + "layer = 5\n", + "sae = hook_name_to_sae[utils.get_act_name('z', layer)]\n", + "model.add_sae(sae)\n", + "print(\"Attached SAEs after add_sae\", model.acts_to_saes)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we can just call the standard HookedTransformer forward, and the sae that we added will automatically be spliced in." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average logit diff with SAEs: 3.6155965328216553\n" + ] + } + ], + "source": [ + "logits_with_saes = model(tokens)\n", + "assert not torch.allclose(original_logits, logits_with_saes, atol=1e-4)\n", + "\n", + "average_logit_diff_with_saes = logits_to_ave_logit_diff(logits_with_saes, answer_tokens)\n", + "print(f\"Average logit diff with SAEs: {average_logit_diff_with_saes}\")\n", + "per_prompt_diff_with_saes = logits_to_ave_logit_diff(logits_with_saes, answer_tokens, per_prompt=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Run with cache" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Similarly, we can also use `logits, cache = model.run_with_cache(tokens)` directly to cache SAE activations:" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.plotly.v1+json": { + "config": { + "plotlyServerURL": "https://plot.ly" + }, + "data": [ + { + "coloraxis": "coloraxis", + "hovertemplate": "Feature Id: %{x}
Prompt: %{y}
color: %{z}", + "name": "0", + "type": "heatmap", + "x": [ + "46", + "345", + "702", + "1372", + "1755", + "1965", + "2457", + "2496", + "2646", + "2999", + "3047", + "4569", + "5132", + "5203", + "5508", + "5940", + "6144", + "6371", + "6515", + "6558", + "6812", + "7092", + "7515", + "7907", + "8063", + "8623", + "8737", + "8768", + "9096", + "9102", + "9186", + "9463", + "9746", + "9913", + "10581", + "10894", + "12109", + "12485", + "12764", + "12866", + "13063", + "13624", + "13707", + "13777", + "14844", + "15050", + "15170", + "15696", + "16178", + "16892", + "17156", + "17259", + "17497", + "17854", + "18043", + "18210", + "18318", + "18385", + "18440", + "18920", + "19183", + "19263", + "19442", + "19524", + "19573", + "20838", + "21151", + "21657", + "22108", + "23578", + "24091", + "24217", + "25792", + "26373", + "26410", + "27535", + "27787", + "27811", + "27960", + "28061", + "28241", + "28242", + "28254", + "28349", + "28977", + "29027", + "29482", + "29603", + "29700", + "29822", + "32177", + "32920", + "33320", + "33730", + "33966", + "34177", + "34334", + "34947", + "35403", + "35425", + "35579", + "35665", + "35815", + "36109", + "36172", + "36451", + "36767", + "36917", + "38570", + "39962", + "40409", + "40418", + "40661", + "41162", + "41185", + "41552", + "42024", + "42161", + "42437", + "42577", + "42882", + "42931", + "43035", + "43414", + "43643", + "43662", + "44203", + "44256", + "44452", + "44652", + "45179", + "45814", + "45984", + "46880", + "47117", + "47170", + "47231", + "47313", + "47680", + "48063", + "48703" + ], + "xaxis": "x", + "yaxis": "y", + "z": [ + [ + 0.23392018675804138, + 0, + 0, + 0.04335343837738037, + 0.44275617599487305, + 0, + 0, + 0.07259953022003174, + 0, + 0.6985604763031006, + 1.262436866760254, + 0, + 0.04656928777694702, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.45666736364364624, + 0.10434150695800781, + 0.30980953574180603, + 0.3319076895713806, + 0, + 0, + 0, + 0, + 1.7836596965789795, + 0, + 0, + 0.142583429813385, + 0.046830952167510986, + 0.3180348575115204, + 0.2927079200744629, + 0.12267106771469116, + 2.5688514709472656, + 0.2917236089706421, + 0.12333670258522034, + 0, + 0.1778419017791748, + 0, + 0.023626387119293213, + 0.02943563461303711, + 0, + 0.048882365226745605, + 0.13625454902648926, + 0, + 0, + 0.2634885013103485, + 0, + 0, + 0, + 0.21662655472755432, + 0, + 0, + 0, + 0.06997489929199219, + 0.006345987319946289, + 0, + 0.16112494468688965, + 0.4190089702606201, + 0, + 2.3819468021392822, + 1.0431660413742065, + 0, + 0.08364987373352051, + 0, + 0, + 0.3451769948005676, + 0.7391350865364075, + 0.4456520080566406, + 0.0019606351852416992, + 0.39914217591285706, + 0, + 0, + 0, + 0.29958274960517883, + 0.44243645668029785, + 0, + 0.1259920299053192, + 0.8349504470825195, + 0.37993764877319336, + 0.2633737325668335, + 0.08324140310287476, + 0, + 0, + 0.10421907901763916, + 0, + 0, + 0, + 0.36972635984420776, + 0, + 0, + 0, + 0, + 0.5578295588493347, + 0, + 0.9233021140098572, + 0, + 0.10010790824890137, + 0, + 0.45082613825798035, + 0, + 0, + 0, + 0.21043556928634644, + 0.12981292605400085, + 0.11557984352111816, + 0, + 0, + 0.17571094632148743, + 0.2823787331581116, + 0.1122598648071289, + 0, + 0, + 0.012049257755279541, + 0, + 0, + 0, + 2.417463541030884, + 0.0547795295715332, + 0.05216425657272339, + 0, + 0.6592545509338379, + 0.003663182258605957, + 0, + 0, + 0.04937589168548584, + 0.025814831256866455, + 0, + 0.8019273281097412, + 0, + 0.10218703746795654 + ], + [ + 0, + 0, + 0.3230956792831421, + 0, + 0, + 0, + 0.026041746139526367, + 0.31818556785583496, + 0, + 0.4900796413421631, + 0.04911249876022339, + 0, + 0, + 0.07309412956237793, + 0.08089971542358398, + 0.17180073261260986, + 0, + 0, + 0, + 0, + 0, + 0, + 2.3956947326660156, + 0, + 0, + 0.15781426429748535, + 0, + 0.5073252320289612, + 0.21765804290771484, + 0, + 0, + 1.618570327758789, + 0, + 0.22485831379890442, + 0.0830467939376831, + 0.7055595517158508, + 0, + 0, + 0, + 0, + 0.23371747136116028, + 0, + 0, + 0.6983060240745544, + 0, + 0, + 0, + 0, + 0.30831730365753174, + 0, + 0.417669415473938, + 0.05292201042175293, + 0, + 0, + 0, + 1.3391070365905762, + 0, + 0.41352108120918274, + 0, + 0, + 0, + 0.037178993225097656, + 0, + 0, + 0, + 0, + 0.2702980041503906, + 0, + 0, + 0.18745100498199463, + 1.3330132961273193, + 0.5793700814247131, + 0.33893001079559326, + 0, + 0.11196631193161011, + 1.720167636871338, + 0.17581266164779663, + 0.42567259073257446, + 0, + 0, + 0.23682871460914612, + 0, + 0, + 0, + 0, + 0, + 1.8280882835388184, + 0.1617840826511383, + 0, + 0.13557660579681396, + 0.5832244157791138, + 0, + 0, + 0.03256487846374512, + 0, + 0, + 0.03892314434051514, + 0, + 0, + 0, + 0.30978846549987793, + 0, + 0, + 0.36915141344070435, + 0, + 0.5477294325828552, + 0, + 0, + 0.06339260935783386, + 0.1851767599582672, + 0.5839155912399292, + 0, + 0, + 0, + 0, + 0, + 0.12337607145309448, + 0, + 0, + 1.0378936529159546, + 0, + 0, + 0, + 0.01616498827934265, + 0.20259439945220947, + 0, + 0, + 0.3087460398674011, + 0.618510365486145, + 0.24435847997665405, + 0, + 0.4668591022491455, + 0.1788468360900879, + 0.200361967086792, + 0, + 0, + 0, + 0, + 0, + 0, + 0.7064645290374756 + ], + [ + 0.2921750843524933, + 0, + 0, + 0.2805737257003784, + 0, + 0, + 0, + 0.3694216012954712, + 0, + 1.1156601905822754, + 1.2807728052139282, + 0, + 0.09175515174865723, + 0, + 0, + 0, + 0.10458803176879883, + 0, + 0.021218180656433105, + 0, + 0, + 0.01699376106262207, + 0.09601330757141113, + 0.054788172245025635, + 0, + 0.030488133430480957, + 0.021512210369110107, + 0.2717320919036865, + 0.29357004165649414, + 0.6420693397521973, + 0.05249035358428955, + 0, + 0.06201601028442383, + 0, + 0.4122554659843445, + 1.821354866027832, + 0.01981794834136963, + 0, + 0.14063221216201782, + 0.05093127489089966, + 0, + 0.32148706912994385, + 0.15257668495178223, + 2.418062686920166, + 0.17348229885101318, + 0.08421656489372253, + 0, + 0.4551248550415039, + 0, + 0.015430927276611328, + 0.24434363842010498, + 0, + 0.06232607364654541, + 0, + 0.04422914981842041, + 0.8720088005065918, + 0.3721686899662018, + 0, + 0, + 0, + 0.340120404958725, + 0, + 0, + 0.07813769578933716, + 0, + 0.0882720947265625, + 0.19706517457962036, + 0.4056885242462158, + 0.19529414176940918, + 0, + 2.928431510925293, + 1.1402223110198975, + 0, + 0.026796698570251465, + 0.0033188462257385254, + 0, + 0.3370524048805237, + 0.47657889127731323, + 0, + 0.10358679294586182, + 0.27619925141334534, + 0, + 0, + 0, + 0.40909066796302795, + 0.2599871754646301, + 0, + 0.275011271238327, + 0.5349323749542236, + 0.07697033882141113, + 0.17431437969207764, + 0, + 0, + 0, + 0.09000074863433838, + 0, + 0, + 0, + 0.276567280292511, + 0, + 0, + 0, + 0, + 0.5655339360237122, + 0, + 0.8971189856529236, + 0, + 0.5199201107025146, + 0, + 0.6301102638244629, + 0.013657361268997192, + 0.04469645023345947, + 0.038062095642089844, + 0.4305816888809204, + 0, + 0.04173767566680908, + 0, + 0, + 0, + 0.8985729217529297, + 0, + 0, + 0, + 0, + 0, + 0.08318889141082764, + 0.006362795829772949, + 2.069222927093506, + 0, + 0.7068352103233337, + 0, + 0.8527798652648926, + 0, + 0, + 0.4707651138305664, + 0, + 0, + 0, + 0.7790955305099487, + 0.021227538585662842, + 0.01846003532409668 + ], + [ + 0, + 0, + 0.2200499176979065, + 0, + 0, + 0, + 0, + 0.2433047890663147, + 0.2504638135433197, + 0.712148904800415, + 0, + 0, + 0, + 0, + 0, + 0.1410943865776062, + 0, + 0, + 0, + 0.11292147636413574, + 0, + 0, + 2.360842704772949, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1.2830760478973389, + 0, + 0, + 0, + 0.6308119893074036, + 0, + 0.4040885865688324, + 0, + 0, + 0, + 0, + 0, + 0.5223236680030823, + 0, + 0, + 0, + 0, + 0.23784160614013672, + 0, + 0.04762387275695801, + 0, + 0, + 0, + 0, + 0.5758676528930664, + 0.01025208830833435, + 0.24556085467338562, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1.1104614734649658, + 1.079118251800537, + 0, + 0, + 0.14462929964065552, + 1.9186956882476807, + 0, + 0.30735498666763306, + 0, + 0, + 0.07669633626937866, + 0, + 0, + 0, + 0, + 0, + 1.3975048065185547, + 0, + 0, + 0.3461639881134033, + 0.5062156915664673, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.19610454142093658, + 0.218009352684021, + 0, + 0, + 0.07953745126724243, + 0, + 0.1416093111038208, + 0, + 0, + 0, + 0.18305465579032898, + 0.10310900211334229, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.45315277576446533, + 0, + 0, + 0, + 0.09076884388923645, + 0, + 0, + 0, + 0, + 0, + 0.04246491193771362, + 0, + 0.1807355284690857, + 0, + 0.3002055883407593, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0.02005404233932495, + 0, + 0, + 0.07601284980773926, + 0, + 0, + 0, + 0.012166053056716919, + 0, + 1.0662918090820312, + 1.4810535907745361, + 0, + 0.014786958694458008, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.1491186022758484, + 0, + 0, + 0, + 0.38226866722106934, + 0.43110355734825134, + 0, + 0, + 0, + 0, + 0, + 1.6819074153900146, + 0, + 0.7939910888671875, + 0.28643298149108887, + 0, + 0, + 0.011532962322235107, + 0, + 1.2869157791137695, + 0, + 0, + 0, + 0, + 0, + 0.16446048021316528, + 0, + 0, + 0, + 0, + 0, + 0, + 0.03375712037086487, + 0, + 0, + 0, + 0.1915181577205658, + 0, + 0, + 0.10225892066955566, + 0, + 0, + 0, + 0.7338485717773438, + 0, + 0, + 1.3715617656707764, + 1.6115869283676147, + 0, + 0.7128411531448364, + 0, + 0, + 0.2161598801612854, + 0.5098914504051208, + 0, + 0, + 0.04084053635597229, + 0, + 0, + 0, + 0.17978456616401672, + 0, + 0, + 0.1365671455860138, + 0.27122950553894043, + 0.2945059537887573, + 0.2824629545211792, + 0, + 0, + 0, + 0.0464092493057251, + 0, + 0, + 0.04672741889953613, + 0.6179839968681335, + 0, + 0, + 0, + 0, + 0.045598745346069336, + 0, + 1.0172381401062012, + 0, + 0.07242608070373535, + 0, + 0.5165215730667114, + 0, + 0, + 0, + 0.5004003047943115, + 0, + 0, + 0, + 0, + 0, + 0.3409433960914612, + 0, + 0.1579979658126831, + 0.09901612997055054, + 0, + 0, + 0, + 0, + 2.413944721221924, + 0, + 0.20971286296844482, + 0.07062971591949463, + 0.26070594787597656, + 0, + 0, + 0, + 0, + 0, + 0.020640969276428223, + 1.0534553527832031, + 0, + 0 + ], + [ + 0, + 0, + 0.046907246112823486, + 0, + 0, + 0, + 0, + 0.20885008573532104, + 0.25957152247428894, + 1.0767037868499756, + 0, + 0, + 0, + 0, + 0, + 0.23976856470108032, + 0, + 0, + 0, + 0, + 0, + 0, + 2.762990951538086, + 0, + 0, + 0, + 0, + 0.29466086626052856, + 0, + 0, + 0.09433537721633911, + 1.2446393966674805, + 0, + 0, + 0, + 0.6668079495429993, + 0, + 0.7482341527938843, + 0, + 0, + 0.005075186491012573, + 0, + 0, + 0.4049275517463684, + 0, + 0, + 0, + 0, + 0.09314888715744019, + 0, + 0, + 0, + 0, + 0, + 0, + 0.4028928279876709, + 0, + 0.3687801659107208, + 0, + 0.10555410385131836, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1.066054105758667, + 1.4596349000930786, + 0, + 0, + 0, + 2.3358588218688965, + 0, + 0.5390753149986267, + 0, + 0, + 0.12931063771247864, + 0, + 0.10619288682937622, + 0, + 0, + 0, + 0.41271400451660156, + 0, + 0, + 0.23865878582000732, + 0.7501264810562134, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.2947666645050049, + 0, + 0, + 0, + 0.05958199501037598, + 0.20450782775878906, + 0, + 0, + 0, + 0.13838836550712585, + 0.13835513591766357, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.45820748805999756, + 0, + 0, + 0, + 0.19962045550346375, + 0, + 0, + 0, + 0, + 0.20416772365570068, + 0.46223968267440796, + 0, + 0.22815394401550293, + 0, + 0.1125795841217041, + 0, + 0, + 0, + 0, + 0, + 0, + 0.3023688793182373 + ], + [ + 0.28365251421928406, + 0, + 0, + 0.41595208644866943, + 0, + 0.15376341342926025, + 0, + 0.22517156600952148, + 0, + 0.7871096134185791, + 1.3084614276885986, + 0.2012956142425537, + 0, + 0, + 0, + 0.2532406449317932, + 0.009012699127197266, + 0, + 0, + 0, + 0, + 0.7235959768295288, + 0.021468758583068848, + 0, + 0, + 0, + 0, + 0.8338297009468079, + 0.3022422790527344, + 0.6702529191970825, + 0.5416026711463928, + 0, + 0, + 0, + 0.2034381628036499, + 1.9052581787109375, + 0, + 0.23752644658088684, + 0, + 0, + 0, + 0.8470145463943481, + 0, + 2.820002555847168, + 0, + 0.16275432705879211, + 0.06714236736297607, + 0.12017238140106201, + 0, + 0, + 0, + 0, + 0.486280620098114, + 0, + 0, + 0.3096342086791992, + 0.3064201772212982, + 0, + 0.09773910045623779, + 0, + 0.4613642394542694, + 0, + 0.021892428398132324, + 0, + 0.18887782096862793, + 0.18538141250610352, + 0, + 0.42975664138793945, + 0.9873132705688477, + 0, + 2.163774013519287, + 1.2928048372268677, + 0, + 0.2320784330368042, + 0.0062233805656433105, + 0, + 1.2478563785552979, + 0.5479208827018738, + 0, + 0.06501156091690063, + 0.3741762936115265, + 0, + 0, + 0.31712013483047485, + 0.5228050947189331, + 0.3981531858444214, + 0, + 0, + 0.4854400157928467, + 0.3341655731201172, + 0.39207732677459717, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.3316766023635864, + 0, + 0, + 0.33435362577438354, + 0.1380615234375, + 0.7183249592781067, + 0.041296958923339844, + 0.7634149193763733, + 0, + 0.4028007984161377, + 0, + 0.6915435791015625, + 0, + 0, + 0, + 0.3831353187561035, + 0.05798754096031189, + 0.15244710445404053, + 0, + 0.03230410814285278, + 0.2039397656917572, + 0.6142292022705078, + 0.15542924404144287, + 0.07628917694091797, + 0.0812273919582367, + 0.15177401900291443, + 0.10224854946136475, + 0, + 0, + 2.8106069564819336, + 0.3994237184524536, + 0.6397127509117126, + 0, + 0.8949670791625977, + 0, + 0, + 0.18832790851593018, + 0.1450880765914917, + 0, + 0, + 0.6900937557220459, + 0, + 0.14745783805847168 + ], + [ + 0.12055802345275879, + 0.023864269256591797, + 0, + 0, + 0, + 0, + 0, + 0.3327372670173645, + 0.1789897382259369, + 1.1445300579071045, + 0, + 0, + 0, + 0, + 0, + 0.4361664652824402, + 0.09996795654296875, + 0.10051405429840088, + 0, + 0.4030296802520752, + 0.06672021746635437, + 0.6339577436447144, + 3.3947582244873047, + 0, + 0, + 0, + 0, + 0.9711236357688904, + 0, + 0.38066884875297546, + 0.4158353805541992, + 1.5344438552856445, + 0, + 0.19816407561302185, + 0, + 0.6646860241889954, + 0, + 0.16733816266059875, + 0, + 0, + 0, + 0.322623074054718, + 0, + 0.7314171195030212, + 0, + 0, + 0, + 0, + 0.043955981731414795, + 0, + 0, + 0, + 0, + 0, + 0, + 0.9436180591583252, + 0, + 0.29259607195854187, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.1570979356765747, + 0, + 0, + 0, + 1.1782727241516113, + 1.2431498765945435, + 0.32878363132476807, + 0, + 0.419150173664093, + 2.3304405212402344, + 0.8566346764564514, + 0, + 0, + 0, + 0.3841046392917633, + 0.10476112365722656, + 0, + 0.18140661716461182, + 0, + 0, + 0.6665420532226562, + 0, + 0, + 0.22877633571624756, + 0.9225524663925171, + 0, + 0.15886402130126953, + 0, + 0, + 0.02094721794128418, + 0, + 0, + 0, + 0.3046541213989258, + 0.2845715284347534, + 0, + 0, + 0.4244043231010437, + 0.164473295211792, + 0.30073386430740356, + 0.7123112678527832, + 0.1730642318725586, + 0, + 0.4041661322116852, + 0.39166414737701416, + 0, + 0, + 0.2103893756866455, + 0.007811635732650757, + 0.010994672775268555, + 0.03914850950241089, + 0, + 0, + 0.8430832624435425, + 0, + 0, + 0, + 0.15830591320991516, + 0.29398930072784424, + 0, + 0, + 0, + 0.5994948148727417, + 0.1704254150390625, + 0, + 0.4673898220062256, + 0, + 0.3204514980316162, + 0, + 0, + 0, + 0, + 0, + 0, + 0.8447363376617432 + ] + ] + } + ], + "layout": { + "coloraxis": { + "cmid": 0, + "colorscale": [ + [ + 0, + "rgb(103,0,31)" + ], + [ + 0.1, + "rgb(178,24,43)" + ], + [ + 0.2, + "rgb(214,96,77)" + ], + [ + 0.3, + "rgb(244,165,130)" + ], + [ + 0.4, + "rgb(253,219,199)" + ], + [ + 0.5, + "rgb(247,247,247)" + ], + [ + 0.6, + "rgb(209,229,240)" + ], + [ + 0.7, + "rgb(146,197,222)" + ], + [ + 0.8, + "rgb(67,147,195)" + ], + [ + 0.9, + "rgb(33,102,172)" + ], + [ + 1, + "rgb(5,48,97)" + ] + ] + }, + "template": { + "data": { + "bar": [ + { + "error_x": { + "color": "#2a3f5f" + }, + "error_y": { + "color": "#2a3f5f" + }, + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "bar" + } + ], + "barpolar": [ + { + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "barpolar" + } + ], + "carpet": [ + { + "aaxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "baxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "type": "carpet" + } + ], + "choropleth": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "choropleth" + } + ], + "contour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "contour" + } + ], + "contourcarpet": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "contourcarpet" + } + ], + "heatmap": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmap" + } + ], + "heatmapgl": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmapgl" + } + ], + "histogram": [ + { + "marker": { + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "histogram" + } + ], + "histogram2d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2d" + } + ], + "histogram2dcontour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2dcontour" + } + ], + "mesh3d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "mesh3d" + } + ], + "parcoords": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "parcoords" + } + ], + "pie": [ + { + "automargin": true, + "type": "pie" + } + ], + "scatter": [ + { + "fillpattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + }, + "type": "scatter" + } + ], + "scatter3d": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatter3d" + } + ], + "scattercarpet": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattercarpet" + } + ], + "scattergeo": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergeo" + } + ], + "scattergl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergl" + } + ], + "scattermapbox": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermapbox" + } + ], + "scatterpolar": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolar" + } + ], + "scatterpolargl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolargl" + } + ], + "scatterternary": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterternary" + } + ], + "surface": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "surface" + } + ], + "table": [ + { + "cells": { + "fill": { + "color": "#EBF0F8" + }, + "line": { + "color": "white" + } + }, + "header": { + "fill": { + "color": "#C8D4E3" + }, + "line": { + "color": "white" + } + }, + "type": "table" + } + ] + }, + "layout": { + "annotationdefaults": { + "arrowcolor": "#2a3f5f", + "arrowhead": 0, + "arrowwidth": 1 + }, + "autotypenumbers": "strict", + "coloraxis": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "colorscale": { + "diverging": [ + [ + 0, + "#8e0152" + ], + [ + 0.1, + "#c51b7d" + ], + [ + 0.2, + "#de77ae" + ], + [ + 0.3, + "#f1b6da" + ], + [ + 0.4, + "#fde0ef" + ], + [ + 0.5, + "#f7f7f7" + ], + [ + 0.6, + "#e6f5d0" + ], + [ + 0.7, + "#b8e186" + ], + [ + 0.8, + "#7fbc41" + ], + [ + 0.9, + "#4d9221" + ], + [ + 1, + "#276419" + ] + ], + "sequential": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "sequentialminus": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ] + }, + "colorway": [ + "#636efa", + "#EF553B", + "#00cc96", + "#ab63fa", + "#FFA15A", + "#19d3f3", + "#FF6692", + "#B6E880", + "#FF97FF", + "#FECB52" + ], + "font": { + "color": "#2a3f5f" + }, + "geo": { + "bgcolor": "white", + "lakecolor": "white", + "landcolor": "#E5ECF6", + "showlakes": true, + "showland": true, + "subunitcolor": "white" + }, + "hoverlabel": { + "align": "left" + }, + "hovermode": "closest", + "mapbox": { + "style": "light" + }, + "paper_bgcolor": "white", + "plot_bgcolor": "#E5ECF6", + "polar": { + "angularaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "radialaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "scene": { + "xaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "yaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "zaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + } + }, + "shapedefaults": { + "line": { + "color": "#2a3f5f" + } + }, + "ternary": { + "aaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "baxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "caxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "title": { + "x": 0.05 + }, + "xaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + }, + "yaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + } + } + }, + "title": { + "text": "Activations of Live SAE features at L5 S2 position per prompt" + }, + "xaxis": { + "anchor": "y", + "constrain": "domain", + "domain": [ + 0, + 1 + ], + "scaleanchor": "y", + "title": { + "text": "Feature Id" + } + }, + "yaxis": { + "anchor": "x", + "autorange": "reversed", + "constrain": "domain", + "domain": [ + 0, + 1 + ], + "title": { + "text": "Prompt" + } + } + } + }, + "text/html": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "layer = 5\n", + "_, cache = model.run_with_cache(tokens)\n", + "s2_pos = 10\n", + "sae_acts = cache[utils.get_act_name('z', layer) + \".hook_sae_acts_post\"][:, s2_pos, :]\n", + "\n", + "live_feature_mask = sae_acts > 0\n", + "live_feature_union = live_feature_mask.any(dim=0)\n", + "\n", + "imshow(\n", + " sae_acts[:, live_feature_union],\n", + " title = \"Activations of Live SAE features at L5 S2 position per prompt\",\n", + " xaxis=\"Feature Id\", yaxis=\"Prompt\",\n", + " x=list(map(str, live_feature_union.nonzero().flatten().tolist())),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Run with hooks" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally we can also use `run_with_hooks` and intervene on the added SAE's activations. To show a more complicated intervention, we'll try path patching the feature from the S-inhibition head's value vectors." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "model.set_use_split_qkv_input(True)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 141/141 [00:05<00:00, 26.94it/s]\n" + ] + }, + { + "data": { + "application/vnd.plotly.v1+json": { + "config": { + "plotlyServerURL": "https://plot.ly" + }, + "data": [ + { + "coloraxis": "coloraxis", + "hovertemplate": "Feature Id: %{x}
Prompt Idx: %{y}
color: %{z}", + "name": "0", + "type": "heatmap", + "x": [ + "46", + "345", + "702", + "1372", + "1755", + "1965", + "2457", + "2496", + "2646", + "2999", + "3047", + "4569", + "5132", + "5203", + "5508", + "5940", + "6144", + "6371", + "6515", + "6558", + "6812", + "7092", + "7515", + "7907", + "8063", + "8623", + "8737", + "8768", + "9096", + "9102", + "9186", + "9463", + "9746", + "9913", + "10581", + "10894", + "12109", + "12485", + "12764", + "12866", + "13063", + "13624", + "13707", + "13777", + "14844", + "15050", + "15170", + "15696", + "16178", + "16892", + "17156", + "17259", + "17497", + "17854", + "18043", + "18210", + "18318", + "18385", + "18440", + "18920", + "19183", + "19263", + "19442", + "19524", + "19573", + "20838", + "21151", + "21657", + "22108", + "23578", + "24091", + "24217", + "25792", + "26373", + "26410", + "27535", + "27787", + "27811", + "27960", + "28061", + "28241", + "28242", + "28254", + "28349", + "28977", + "29027", + "29482", + "29603", + "29700", + "29822", + "32177", + "32920", + "33320", + "33730", + "33966", + "34177", + "34334", + "34947", + "35403", + "35425", + "35579", + "35665", + "35815", + "36109", + "36172", + "36451", + "36767", + "36917", + "38570", + "39962", + "40409", + "40418", + "40661", + "41162", + "41185", + "41552", + "42024", + "42161", + "42437", + "42577", + "42882", + "42931", + "43035", + "43414", + "43643", + "43662", + "44203", + "44256", + "44452", + "44652", + "45179", + "45814", + "45984", + "46880", + "47117", + "47170", + "47231", + "47313", + "47680", + "48063", + "48703" + ], + "xaxis": "x", + "yaxis": "y", + "z": [ + [ + 0.0005645751953125, + 0.0000057220458984375, + 0.0000057220458984375, + 0.000339508056640625, + -0.003261566162109375, + 0.0000057220458984375, + 0.0000057220458984375, + 0.00069427490234375, + 0.0000057220458984375, + 0.0016155242919921875, + -0.09088897705078125, + 0.0000057220458984375, + 0.00011444091796875, + 0.0000057220458984375, + 0.0000057220458984375, + 0.0000057220458984375, + 0.0000057220458984375, + 0.0000057220458984375, + 0.0000057220458984375, + 0.0000057220458984375, + 0.0000057220458984375, + 0.0000057220458984375, + 0.0000057220458984375, + 0.0000057220458984375, + 0.0000057220458984375, + 0.0000057220458984375, + 0.0000057220458984375, + -0.009515762329101562, + -0.0022525787353515625, + 0.0031604766845703125, + -0.0020704269409179688, + 0.0000057220458984375, + 0.0000057220458984375, + 0.0000057220458984375, + 0.0000057220458984375, + -0.013577461242675781, + 0.0000057220458984375, + 0.0000057220458984375, + -0.0017032623291015625, + 0.0002880096435546875, + -0.00020503997802734375, + -0.0016231536865234375, + 0.00037860870361328125, + -0.0098114013671875, + -0.002185821533203125, + -0.0008878707885742188, + 0.0000057220458984375, + 0.0002346038818359375, + 0.0000057220458984375, + -0.000354766845703125, + 0.00036334991455078125, + 0.0000057220458984375, + -0.000988006591796875, + -0.00044918060302734375, + 0.0000057220458984375, + 0.0000057220458984375, + 0.005593299865722656, + 0.0000057220458984375, + 0.0000057220458984375, + 0.0000057220458984375, + -0.005214691162109375, + 0.0000057220458984375, + 0.0000057220458984375, + 0.0000057220458984375, + -0.000789642333984375, + 0.00010585784912109375, + 0.0000057220458984375, + -0.0059051513671875, + 0.0011091232299804688, + 0.0000057220458984375, + 0.026823997497558594, + 0.019052505493164062, + 0.0000057220458984375, + 0.0000152587890625, + 0.0000057220458984375, + 0.0000057220458984375, + 0.0033597946166992188, + -0.020666122436523438, + -0.0041141510009765625, + -0.000011444091796875, + 0.00130462646484375, + 0.0000057220458984375, + 0.0000057220458984375, + 0.0000057220458984375, + -0.01567840576171875, + 0.006500244140625, + 0.0000057220458984375, + 0.002086639404296875, + 0.00576019287109375, + 0.004245758056640625, + 0.006832122802734375, + 0.0006284713745117188, + 0.0000057220458984375, + 0.0000057220458984375, + -0.0009737014770507812, + 0.0000057220458984375, + 0.0000057220458984375, + 0.0000057220458984375, + -0.0040988922119140625, + 0.0000057220458984375, + 0.0000057220458984375, + 0.0000057220458984375, + 0.0000057220458984375, + -0.003326416015625, + 0.0000057220458984375, + 0.020755767822265625, + 0.0000057220458984375, + -0.0008373260498046875, + 0.0000057220458984375, + 0.007825851440429688, + 0.0000057220458984375, + 0.0000057220458984375, + 0.0000057220458984375, + -0.002574920654296875, + 0.00151824951171875, + -0.00008678436279296875, + 0.0000057220458984375, + 0.0000057220458984375, + 0.001171112060546875, + -0.02040863037109375, + -0.0014247894287109375, + 0.0000057220458984375, + 0.0000057220458984375, + 0.00003814697265625, + 0.0000057220458984375, + 0.0000057220458984375, + 0.0000057220458984375, + -0.3322334289550781, + 0.000579833984375, + 0.001293182373046875, + 0.0000057220458984375, + 0.0066661834716796875, + 0.0000171661376953125, + 0.0000057220458984375, + 0.0000057220458984375, + 0.0005435943603515625, + 0.00032806396484375, + 0.0000057220458984375, + 0.023120880126953125, + 0.0000057220458984375, + -0.0017566680908203125 + ], + [ + 0.00000762939453125, + 0.00000762939453125, + 0.0040073394775390625, + 0.00000762939453125, + 0.00000762939453125, + 0.00000762939453125, + 0.0022735595703125, + -0.0012683868408203125, + 0.00000762939453125, + 0.017993927001953125, + 0.011075973510742188, + 0.00000762939453125, + 0.00000762939453125, + -0.001407623291015625, + -0.000270843505859375, + -0.010431289672851562, + 0.00000762939453125, + 0.00000762939453125, + 0.00000762939453125, + 0.00000762939453125, + 0.00000762939453125, + 0.00000762939453125, + -0.6347770690917969, + 0.00000762939453125, + 0.00000762939453125, + 0.0005435943603515625, + 0.00000762939453125, + 0.09274864196777344, + 0.008495330810546875, + 0.00000762939453125, + 0.00000762939453125, + -0.08464431762695312, + 0.00000762939453125, + 0.028835296630859375, + 0.01250457763671875, + 0.029806137084960938, + 0.00000762939453125, + 0.00000762939453125, + 0.00000762939453125, + 0.00000762939453125, + 0.012714385986328125, + 0.00000762939453125, + 0.00000762939453125, + -0.0004444122314453125, + 0.00000762939453125, + 0.00000762939453125, + 0.00000762939453125, + 0.00000762939453125, + 0.003757476806640625, + 0.00000762939453125, + 0.0025272369384765625, + 0.0013427734375, + 0.00000762939453125, + 0.00000762939453125, + 0.00000762939453125, + 0.07260704040527344, + 0.00000762939453125, + 0.01149749755859375, + 0.00000762939453125, + 0.00000762939453125, + 0.00000762939453125, + -0.000213623046875, + 0.00000762939453125, + 0.00000762939453125, + 0.00000762939453125, + 0.00000762939453125, + -0.016370773315429688, + 0.00000762939453125, + 0.00000762939453125, + -0.00792694091796875, + 0.03365135192871094, + -0.004932403564453125, + 0.005069732666015625, + 0.00000762939453125, + 0.0031223297119140625, + -0.5932121276855469, + -0.0007534027099609375, + 0.05148506164550781, + 0.00000762939453125, + 0.00000762939453125, + 0.014024734497070312, + 0.00000762939453125, + 0.00000762939453125, + 0.00000762939453125, + 0.00000762939453125, + 0.00000762939453125, + -0.11317634582519531, + -0.0026416778564453125, + 0.00000762939453125, + -0.006038665771484375, + 0.00672149658203125, + 0.00000762939453125, + 0.00000762939453125, + 0.000064849853515625, + 0.00000762939453125, + 0.00000762939453125, + 0.0005397796630859375, + 0.00000762939453125, + 0.00000762939453125, + 0.00000762939453125, + -0.0024967193603515625, + 0.00000762939453125, + 0.00000762939453125, + 0.016933441162109375, + 0.00000762939453125, + -0.0049343109130859375, + 0.00000762939453125, + 0.00000762939453125, + -0.00244140625, + -0.00624847412109375, + 0.018770217895507812, + 0.00000762939453125, + 0.00000762939453125, + 0.00000762939453125, + 0.00000762939453125, + 0.00000762939453125, + -0.001132965087890625, + 0.00000762939453125, + 0.00000762939453125, + 0.1962738037109375, + 0.00000762939453125, + 0.00000762939453125, + 0.00000762939453125, + -0.0005283355712890625, + 0.0070934295654296875, + 0.00000762939453125, + 0.00000762939453125, + 0.10946464538574219, + 0.05410957336425781, + -0.0026397705078125, + 0.00000762939453125, + 0.005107879638671875, + 0.006359100341796875, + -0.04090118408203125, + 0.00000762939453125, + 0.00000762939453125, + 0.00000762939453125, + 0.00000762939453125, + 0.00000762939453125, + 0.00000762939453125, + 0.06792449951171875 + ], + [ + 0.0032672882080078125, + 0.00000667572021484375, + 0.00000667572021484375, + 0.0026044845581054688, + 0.00000667572021484375, + 0.00000667572021484375, + 0.00000667572021484375, + -0.0013751983642578125, + 0.00000667572021484375, + 0.018096923828125, + -0.29747962951660156, + 0.00000667572021484375, + 0.00159454345703125, + 0.00000667572021484375, + 0.00000667572021484375, + 0.00000667572021484375, + -0.00185394287109375, + 0.00000667572021484375, + 0.000064849853515625, + 0.00000667572021484375, + 0.00000667572021484375, + 0.0004253387451171875, + 0.02138805389404297, + 0.000370025634765625, + 0.00000667572021484375, + -0.0002880096435546875, + 0.000560760498046875, + -0.03230476379394531, + -0.02060699462890625, + 0.020964622497558594, + -0.0022487640380859375, + 0.00000667572021484375, + 0.001964569091796875, + 0.00000667572021484375, + -0.07773113250732422, + -0.042862892150878906, + 0.00027751922607421875, + 0.00000667572021484375, + -0.0020580291748046875, + 0.001407623291015625, + 0.00000667572021484375, + -0.0008306503295898438, + 0.00371551513671875, + -0.08299636840820312, + -0.0030012130737304688, + -0.0021905899047851562, + 0.00000667572021484375, + 0.011617660522460938, + 0.00000667572021484375, + -0.0000152587890625, + 0.005359649658203125, + 0.00000667572021484375, + -0.0042018890380859375, + 0.00000667572021484375, + 0.0008802413940429688, + -0.049579620361328125, + 0.010822296142578125, + 0.00000667572021484375, + 0.00000667572021484375, + 0.00000667572021484375, + -0.014369964599609375, + 0.00000667572021484375, + 0.00000667572021484375, + -0.0016632080078125, + 0.00000667572021484375, + 0.0035800933837890625, + 0.024021148681640625, + -0.04512596130371094, + -0.0006885528564453125, + 0.00000667572021484375, + 0.013338088989257812, + 0.06371307373046875, + 0.00000667572021484375, + 0.000629425048828125, + 0.00002002716064453125, + 0.00000667572021484375, + 0.015112876892089844, + -0.05301094055175781, + 0.00000667572021484375, + -0.0011320114135742188, + 0.0012521743774414062, + 0.00000667572021484375, + 0.00000667572021484375, + 0.00000667572021484375, + -0.038700103759765625, + -0.0035238265991210938, + 0.00000667572021484375, + 0.00608062744140625, + -0.011157035827636719, + 0.004566192626953125, + 0.0062274932861328125, + 0.00000667572021484375, + 0.00000667572021484375, + 0.00000667572021484375, + -0.0015010833740234375, + 0.00000667572021484375, + 0.00000667572021484375, + 0.00000667572021484375, + -0.010572433471679688, + 0.00000667572021484375, + 0.00000667572021484375, + 0.00000667572021484375, + 0.00000667572021484375, + -0.016614913940429688, + 0.00000667572021484375, + 0.030905723571777344, + 0.00000667572021484375, + -0.015107154846191406, + 0.00000667572021484375, + 0.012714385986328125, + -0.0009021759033203125, + -0.00067138671875, + 0.0006847381591796875, + -0.005970954895019531, + 0.00000667572021484375, + 0.000392913818359375, + 0.00000667572021484375, + 0.00000667572021484375, + 0.00000667572021484375, + -0.20943737030029297, + 0.00000667572021484375, + 0.00000667572021484375, + 0.00000667572021484375, + 0.00000667572021484375, + 0.00000667572021484375, + 0.0024538040161132812, + -0.00016117095947265625, + -0.6926145553588867, + 0.00000667572021484375, + -0.006705284118652344, + 0.00000667572021484375, + 0.013433456420898438, + 0.00000667572021484375, + 0.00000667572021484375, + 0.0039653778076171875, + 0.00000667572021484375, + 0.00000667572021484375, + 0.00000667572021484375, + 0.05192756652832031, + -0.00046539306640625, + -0.0010156631469726562 + ], + [ + -0.00000667572021484375, + -0.00000667572021484375, + 0.0073337554931640625, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + 0.0017852783203125, + 0.021762847900390625, + 0.023838043212890625, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.0093231201171875, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + 0.00185394287109375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.7318296432495117, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.06693649291992188, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + 0.04135417938232422, + -0.00000667572021484375, + 0.0012073516845703125, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.0023708343505859375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + 0.00597381591796875, + -0.00000667572021484375, + 0.0001049041748046875, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + 0.04203224182128906, + -0.000133514404296875, + 0.0032367706298828125, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + 0.053966522216796875, + -0.017469406127929688, + -0.00000667572021484375, + -0.00000667572021484375, + 0.0032787322998046875, + -0.8294486999511719, + -0.00000667572021484375, + 0.042545318603515625, + -0.00000667572021484375, + -0.00000667572021484375, + 0.006573677062988281, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.1314229965209961, + -0.00000667572021484375, + -0.00000667572021484375, + -0.022655487060546875, + 0.0008211135864257812, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.019756317138671875, + -0.0028676986694335938, + -0.00000667572021484375, + -0.00000667572021484375, + 0.0034084320068359375, + -0.00000667572021484375, + 0.0000171661376953125, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.0022497177124023438, + 0.00191497802734375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + 0.09851455688476562, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.003956794738769531, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + 0.0011348724365234375, + -0.00000667572021484375, + 0.0007839202880859375, + -0.00000667572021484375, + -0.0783843994140625, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375, + -0.00000667572021484375 + ], + [ + -0.00021839141845703125, + 0.00000286102294921875, + 0.00000286102294921875, + -0.00017833709716796875, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00000286102294921875, + -0.00004863739013671875, + 0.00000286102294921875, + 0.0024118423461914062, + -0.1688375473022461, + 0.00000286102294921875, + 0.0005617141723632812, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00000286102294921875, + -0.0027265548706054688, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00000286102294921875, + -0.009179115295410156, + 0.011872291564941406, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00000286102294921875, + 0.006281852722167969, + 0.00000286102294921875, + 0.011416435241699219, + 0.014454841613769531, + 0.00000286102294921875, + 0.00000286102294921875, + -0.00018596649169921875, + 0.00000286102294921875, + 0.012002944946289062, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00000286102294921875, + -0.0023813247680664062, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00000286102294921875, + 0.000225067138671875, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00000286102294921875, + -0.0033779144287109375, + 0.00000286102294921875, + 0.00000286102294921875, + -0.0017099380493164062, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00000286102294921875, + -0.05732154846191406, + 0.00000286102294921875, + 0.00000286102294921875, + 0.016089439392089844, + 0.07070255279541016, + 0.00000286102294921875, + 0.014483451843261719, + 0.00000286102294921875, + 0.00000286102294921875, + 0.0017747879028320312, + -0.024786949157714844, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00012302398681640625, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00000286102294921875, + -0.0092620849609375, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00185394287109375, + -0.00025177001953125, + 0.008860588073730469, + 0.006030082702636719, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00017833709716796875, + 0.00000286102294921875, + 0.00000286102294921875, + -0.001644134521484375, + 0.0026140213012695312, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00000286102294921875, + -0.0013418197631835938, + 0.00000286102294921875, + 0.037514686584472656, + 0.00000286102294921875, + -0.00038433074951171875, + 0.00000286102294921875, + 0.01964282989501953, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00000286102294921875, + 0.005845069885253906, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00000286102294921875, + -0.04890918731689453, + 0.00000286102294921875, + 0.008494377136230469, + -0.00026988983154296875, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00000286102294921875, + -0.37475109100341797, + 0.00000286102294921875, + 0.004479408264160156, + -0.0015649795532226562, + 0.00385284423828125, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00000286102294921875, + 0.00030803680419921875, + 0.06992149353027344, + 0.00000286102294921875, + 0.00000286102294921875 + ], + [ + 0.0000019073486328125, + 0.0000019073486328125, + 0.0018415451049804688, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0016222000122070312, + 0.023705482482910156, + 0.07090950012207031, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + -0.021169662475585938, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + -1.3031587600708008, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.08781909942626953, + 0.0000019073486328125, + 0.0000019073486328125, + 0.016541481018066406, + -0.10686969757080078, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.04713726043701172, + 0.0000019073486328125, + 0.002704620361328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.00046062469482421875, + 0.0000019073486328125, + 0.0000019073486328125, + -0.01665210723876953, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0031337738037109375, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0426177978515625, + 0.0000019073486328125, + 0.018036842346191406, + 0.0000019073486328125, + -0.001964569091796875, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0788869857788086, + -0.03188610076904297, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + -1.4886322021484375, + 0.0000019073486328125, + 0.0885171890258789, + 0.0000019073486328125, + 0.0000019073486328125, + 0.01448822021484375, + 0.0000019073486328125, + -0.0066547393798828125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + -0.045001983642578125, + 0.0000019073486328125, + 0.0000019073486328125, + -0.017113685607910156, + 0.010157585144042969, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + -0.0030698776245117188, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + -0.0001583099365234375, + -0.004227638244628906, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + -0.008556365966796875, + 0.007357597351074219, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.13220977783203125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + -0.013454437255859375, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.033707618713378906, + 0.006083488464355469, + 0.0000019073486328125, + 0.0014142990112304688, + 0.0000019073486328125, + -0.04172039031982422, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.0000019073486328125, + 0.03891944885253906 + ], + [ + 0.0013017654418945312, + 0.00000858306884765625, + 0.00000858306884765625, + 0.0019664764404296875, + 0.00000858306884765625, + 0.0035953521728515625, + 0.00000858306884765625, + 0.0006504058837890625, + 0.00000858306884765625, + 0.0031061172485351562, + -0.07722282409667969, + -0.0011444091796875, + 0.00000858306884765625, + 0.00000858306884765625, + 0.00000858306884765625, + 0.0056209564208984375, + 0.00003147125244140625, + 0.00000858306884765625, + 0.00000858306884765625, + 0.00000858306884765625, + 0.00000858306884765625, + 0.015537261962890625, + 0.001983642578125, + 0.00000858306884765625, + 0.00000858306884765625, + 0.00000858306884765625, + 0.00000858306884765625, + -0.025580406188964844, + -0.005356788635253906, + 0.016262054443359375, + -0.005573272705078125, + 0.00000858306884765625, + 0.00000858306884765625, + 0.00000858306884765625, + -0.0113525390625, + 0.013624191284179688, + 0.00000858306884765625, + 0.000110626220703125, + 0.00000858306884765625, + 0.00000858306884765625, + 0.00000858306884765625, + 0.000293731689453125, + 0.00000858306884765625, + 0.026404380798339844, + 0.00000858306884765625, + 0.0005817413330078125, + 0.00007343292236328125, + 0.0010223388671875, + 0.00000858306884765625, + 0.00000858306884765625, + 0.00000858306884765625, + 0.00000858306884765625, + -0.009862899780273438, + 0.00000858306884765625, + 0.00000858306884765625, + -0.006870269775390625, + 0.00435638427734375, + 0.00000858306884765625, + 0.000232696533203125, + 0.00000858306884765625, + -0.00616455078125, + 0.00000858306884765625, + 0.00033283233642578125, + 0.00000858306884765625, + -0.0016880035400390625, + 0.00286102294921875, + 0.00000858306884765625, + -0.01665496826171875, + 0.008039474487304688, + 0.00000858306884765625, + 0.03484916687011719, + 0.018899917602539062, + 0.00000858306884765625, + 0.00034809112548828125, + -0.0000095367431640625, + 0.00000858306884765625, + 0.022369384765625, + -0.00615692138671875, + 0.00000858306884765625, + -0.00008392333984375, + -0.0018634796142578125, + 0.00000858306884765625, + 0.00000858306884765625, + -0.0065765380859375, + -0.00798797607421875, + 0.007740974426269531, + 0.00000858306884765625, + 0.00000858306884765625, + 0.0047855377197265625, + 0.00484466552734375, + 0.006256103515625, + 0.00000858306884765625, + 0.00000858306884765625, + 0.00000858306884765625, + 0.00000858306884765625, + 0.00000858306884765625, + 0.00000858306884765625, + 0.00000858306884765625, + -0.005949974060058594, + 0.00000858306884765625, + 0.00000858306884765625, + -0.0056934356689453125, + -0.0057353973388671875, + -0.005535125732421875, + 0.00028228759765625, + 0.0137786865234375, + 0.00000858306884765625, + 0.0026874542236328125, + 0.00000858306884765625, + 0.01714324951171875, + 0.00000858306884765625, + 0.00000858306884765625, + 0.00000858306884765625, + 0.00165557861328125, + 0.0006313323974609375, + -0.00090789794921875, + 0.00000858306884765625, + 0.00016021728515625, + 0.00311279296875, + -0.04284191131591797, + -0.00058746337890625, + 0.0028972625732421875, + -0.001148223876953125, + 0.0013751983642578125, + -0.0005426406860351562, + 0.00000858306884765625, + 0.00000858306884765625, + -0.29439735412597656, + 0.0019617080688476562, + 0.018915176391601562, + 0.00000858306884765625, + 0.009466171264648438, + 0.00000858306884765625, + 0.00000858306884765625, + 0.0011997222900390625, + 0.001117706298828125, + 0.00000858306884765625, + 0.00000858306884765625, + 0.02146148681640625, + 0.00000858306884765625, + -0.0012531280517578125 + ], + [ + 0.005249977111816406, + 0.0015926361083984375, + -0.00000286102294921875, + -0.00000286102294921875, + -0.00000286102294921875, + -0.00000286102294921875, + -0.00000286102294921875, + 0.0021581649780273438, + 0.01883697509765625, + 0.0733022689819336, + -0.00000286102294921875, + -0.00000286102294921875, + -0.00000286102294921875, + -0.00000286102294921875, + -0.00000286102294921875, + -0.04300212860107422, + 0.0030794143676757812, + -0.0017910003662109375, + -0.00000286102294921875, + 0.016645431518554688, + -0.021103858947753906, + 0.013091087341308594, + -1.6041021347045898, + -0.00000286102294921875, + -0.00000286102294921875, + -0.00000286102294921875, + -0.00000286102294921875, + 0.3691072463989258, + -0.00000286102294921875, + -0.01113128662109375, + 0.09581279754638672, + -0.11300373077392578, + -0.00000286102294921875, + 0.047149658203125, + -0.00000286102294921875, + 0.053336143493652344, + -0.00000286102294921875, + 0.004380226135253906, + -0.00000286102294921875, + -0.00000286102294921875, + -0.00000286102294921875, + -0.008252143859863281, + -0.00000286102294921875, + -0.018776893615722656, + -0.00000286102294921875, + -0.00000286102294921875, + -0.00000286102294921875, + -0.00000286102294921875, + 0.0016984939575195312, + -0.00000286102294921875, + -0.00000286102294921875, + -0.00000286102294921875, + -0.00000286102294921875, + -0.00000286102294921875, + -0.00000286102294921875, + 0.10451030731201172, + -0.00000286102294921875, + 0.010519981384277344, + -0.00000286102294921875, + -0.00000286102294921875, + -0.00000286102294921875, + -0.00000286102294921875, + -0.00000286102294921875, + -0.00000286102294921875, + -0.00000286102294921875, + -0.00000286102294921875, + -0.014172554016113281, + -0.00000286102294921875, + -0.00000286102294921875, + -0.00000286102294921875, + 0.07860374450683594, + -0.047211647033691406, + 0.010329246520996094, + -0.00000286102294921875, + 0.02579212188720703, + -1.5303049087524414, + -0.020979881286621094, + -0.00000286102294921875, + -0.00000286102294921875, + -0.00000286102294921875, + 0.05430316925048828, + 0.006442070007324219, + -0.00000286102294921875, + 0.035637855529785156, + -0.00000286102294921875, + -0.00000286102294921875, + -0.0784912109375, + -0.00000286102294921875, + -0.00000286102294921875, + -0.020351409912109375, + 0.02591228485107422, + -0.00000286102294921875, + 0.0030069351196289062, + -0.00000286102294921875, + -0.00000286102294921875, + 0.0012063980102539062, + -0.00000286102294921875, + -0.00000286102294921875, + -0.00000286102294921875, + -0.04748249053955078, + -0.00510406494140625, + -0.00000286102294921875, + -0.00000286102294921875, + 0.03345203399658203, + -0.0017213821411132812, + -0.008072853088378906, + 0.014155387878417969, + -0.003909111022949219, + -0.00000286102294921875, + -0.02114105224609375, + 0.021615028381347656, + -0.00000286102294921875, + -0.00000286102294921875, + 0.011925697326660156, + 0.0005092620849609375, + 0.000263214111328125, + -0.00007343292236328125, + -0.00000286102294921875, + -0.00000286102294921875, + 0.2987813949584961, + -0.00000286102294921875, + -0.00000286102294921875, + -0.00000286102294921875, + -0.011395454406738281, + 0.01917552947998047, + -0.00000286102294921875, + -0.00000286102294921875, + -0.00000286102294921875, + 0.12213993072509766, + 0.0026998519897460938, + -0.00000286102294921875, + 0.009751319885253906, + -0.00000286102294921875, + -0.12412357330322266, + -0.00000286102294921875, + -0.00000286102294921875, + -0.00000286102294921875, + -0.00000286102294921875, + -0.00000286102294921875, + -0.00000286102294921875, + 0.15506553649902344 + ] + ] + } + ], + "layout": { + "coloraxis": { + "cmid": 0, + "colorscale": [ + [ + 0, + "rgb(103,0,31)" + ], + [ + 0.1, + "rgb(178,24,43)" + ], + [ + 0.2, + "rgb(214,96,77)" + ], + [ + 0.3, + "rgb(244,165,130)" + ], + [ + 0.4, + "rgb(253,219,199)" + ], + [ + 0.5, + "rgb(247,247,247)" + ], + [ + 0.6, + "rgb(209,229,240)" + ], + [ + 0.7, + "rgb(146,197,222)" + ], + [ + 0.8, + "rgb(67,147,195)" + ], + [ + 0.9, + "rgb(33,102,172)" + ], + [ + 1, + "rgb(5,48,97)" + ] + ] + }, + "template": { + "data": { + "bar": [ + { + "error_x": { + "color": "#2a3f5f" + }, + "error_y": { + "color": "#2a3f5f" + }, + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "bar" + } + ], + "barpolar": [ + { + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "barpolar" + } + ], + "carpet": [ + { + "aaxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "baxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "type": "carpet" + } + ], + "choropleth": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "choropleth" + } + ], + "contour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "contour" + } + ], + "contourcarpet": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "contourcarpet" + } + ], + "heatmap": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmap" + } + ], + "heatmapgl": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmapgl" + } + ], + "histogram": [ + { + "marker": { + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "histogram" + } + ], + "histogram2d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2d" + } + ], + "histogram2dcontour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2dcontour" + } + ], + "mesh3d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "mesh3d" + } + ], + "parcoords": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "parcoords" + } + ], + "pie": [ + { + "automargin": true, + "type": "pie" + } + ], + "scatter": [ + { + "fillpattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + }, + "type": "scatter" + } + ], + "scatter3d": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatter3d" + } + ], + "scattercarpet": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattercarpet" + } + ], + "scattergeo": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergeo" + } + ], + "scattergl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergl" + } + ], + "scattermapbox": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermapbox" + } + ], + "scatterpolar": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolar" + } + ], + "scatterpolargl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolargl" + } + ], + "scatterternary": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterternary" + } + ], + "surface": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "surface" + } + ], + "table": [ + { + "cells": { + "fill": { + "color": "#EBF0F8" + }, + "line": { + "color": "white" + } + }, + "header": { + "fill": { + "color": "#C8D4E3" + }, + "line": { + "color": "white" + } + }, + "type": "table" + } + ] + }, + "layout": { + "annotationdefaults": { + "arrowcolor": "#2a3f5f", + "arrowhead": 0, + "arrowwidth": 1 + }, + "autotypenumbers": "strict", + "coloraxis": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "colorscale": { + "diverging": [ + [ + 0, + "#8e0152" + ], + [ + 0.1, + "#c51b7d" + ], + [ + 0.2, + "#de77ae" + ], + [ + 0.3, + "#f1b6da" + ], + [ + 0.4, + "#fde0ef" + ], + [ + 0.5, + "#f7f7f7" + ], + [ + 0.6, + "#e6f5d0" + ], + [ + 0.7, + "#b8e186" + ], + [ + 0.8, + "#7fbc41" + ], + [ + 0.9, + "#4d9221" + ], + [ + 1, + "#276419" + ] + ], + "sequential": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "sequentialminus": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ] + }, + "colorway": [ + "#636efa", + "#EF553B", + "#00cc96", + "#ab63fa", + "#FFA15A", + "#19d3f3", + "#FF6692", + "#B6E880", + "#FF97FF", + "#FECB52" + ], + "font": { + "color": "#2a3f5f" + }, + "geo": { + "bgcolor": "white", + "lakecolor": "white", + "landcolor": "#E5ECF6", + "showlakes": true, + "showland": true, + "subunitcolor": "white" + }, + "hoverlabel": { + "align": "left" + }, + "hovermode": "closest", + "mapbox": { + "style": "light" + }, + "paper_bgcolor": "white", + "plot_bgcolor": "#E5ECF6", + "polar": { + "angularaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "radialaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "scene": { + "xaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "yaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "zaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + } + }, + "shapedefaults": { + "line": { + "color": "#2a3f5f" + } + }, + "ternary": { + "aaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "baxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "caxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "title": { + "x": 0.05 + }, + "xaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + }, + "yaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + } + } + }, + "title": { + "text": "Change in logit diff when path patching features from S_inhibition heads values per prompts" + }, + "xaxis": { + "anchor": "y", + "constrain": "domain", + "domain": [ + 0, + 1 + ], + "scaleanchor": "y", + "title": { + "text": "Feature Id" + } + }, + "yaxis": { + "anchor": "x", + "autorange": "reversed", + "constrain": "domain", + "domain": [ + 0, + 1 + ], + "title": { + "text": "Prompt Idx" + } + } + } + }, + "text/html": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def path_patch_v_input(v_input, hook, feature_dirs, pos, head_index):\n", + " v_input[:, pos, head_index, :] = v_input[:, pos, head_index, :] - feature_dirs\n", + " return v_input\n", + "\n", + "\n", + "s_inhib_heads = [(7, 3), (7, 9), (8,6), (8,10)]\n", + "\n", + "results = torch.zeros(tokens.shape[0], all_live_features.shape[0])\n", + "\n", + "W_O_cat = einops.rearrange(\n", + " model.W_O,\n", + " \"n_layers n_heads d_head d_model -> n_layers (n_heads d_head) d_model\"\n", + ")\n", + "\n", + "for feature_id in tqdm.tqdm(all_live_features):\n", + " feature_id = feature_id.item()\n", + " feature_acts = cache[utils.get_act_name('z', abl_layer) + \".hook_sae_acts_post\"][:, abl_pos, feature_id] # [batch]\n", + " feature_dirs = (feature_acts.unsqueeze(-1) * sae.W_dec[feature_id]) @ W_O_cat[abl_layer]\n", + " hook_fns = [\n", + " (utils.get_act_name('v_input', layer), partial(path_patch_v_input, feature_dirs=feature_dirs, pos=abl_pos, head_index=head)) for (layer, head) in s_inhib_heads\n", + " ]\n", + " path_patched_logits = model.run_with_hooks(\n", + " tokens,\n", + " return_type=\"logits\",\n", + " fwd_hooks=hook_fns\n", + " )\n", + "\n", + " path_patched_logit_diff = logits_to_ave_logit_diff(path_patched_logits, answer_tokens, per_prompt=True)\n", + " results[:, fid_to_idx[feature_id]] = path_patched_logit_diff - clean_sae_baseline_per_prompt\n", + "\n", + "imshow(\n", + " results, \n", + " title=f\"Change in logit diff when path patching features from S_inhibition heads values per prompts\",\n", + " xaxis=\"Feature Id\", yaxis=\"Prompt Idx\", x=list(map(str, all_live_features.tolist()))\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Reset SAEs" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "One major footgun is forgetting about an SAE that you previously attached with `add_sae`. Similar to TransformerLens `reset_hooks`, you can always reset SAEs you've added with `model.reset_saes()`. You can also pass in a list of activation names to only reset a subset of attached SAEs." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Attached SAEs before reset_saes: {'blocks.5.attn.hook_z': HookedSAE(\n", + " (hook_sae_input): HookPoint()\n", + " (hook_sae_acts_pre): HookPoint()\n", + " (hook_sae_acts_post): HookPoint()\n", + " (hook_sae_recons): HookPoint()\n", + " (hook_sae_error): HookPoint()\n", + " (hook_sae_output): HookPoint()\n", + ")}\n", + "Attached SAEs after reset_saes: {}\n" + ] + } + ], + "source": [ + "print(\"Attached SAEs before reset_saes:\", model.acts_to_saes)\n", + "model.reset_saes()\n", + "print(\"Attached SAEs after reset_saes:\", model.acts_to_saes)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that the HookedSAETransformer API is generally designed to closely match TransformerLens hooks API." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Error Nodes" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Recent exciting work from [Marks et al.](https://arxiv.org/abs/2403.19647v2) demonstrated the use of \"error nodes\" in SAE circuit analysis. The idea is that for some input activation x, SAE(x) = x_reconstruct is an approximation of x, but we can define an error_term such that x = x_reconstruct + error_term.\n", + "\n", + "This seems useful: instead of replacing x with x_reconstruct, which might break everything and make our circuit analysis janky, we can just re-write x as a function of the SAE features, bias, and error term, which gives us access to all of the SAE features but without breaking performance. \n", + "\n", + "Additionally, we can compare interventions on SAE features to the same intervention on the error term to get a better sense of how much the SAE features have actually captured.\n", + "\n", + "To use error terms with HookedSAEs, you can set `hooked_sae.cfg.use_error_term = True`, or initialize it to True in the config. Note HookedSAEConfig sets this to False by default." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Attached SAEs after adding l5_sae_with_error: {'blocks.5.attn.hook_z': HookedSAE(\n", + " (hook_sae_input): HookPoint()\n", + " (hook_sae_acts_pre): HookPoint()\n", + " (hook_sae_acts_post): HookPoint()\n", + " (hook_sae_recons): HookPoint()\n", + " (hook_sae_error): HookPoint()\n", + " (hook_sae_output): HookPoint()\n", + ")}\n" + ] + } + ], + "source": [ + "import copy\n", + "l5_sae = hook_name_to_sae[utils.get_act_name('z', 5)]\n", + "l5_sae_with_error = copy.deepcopy(l5_sae)\n", + "l5_sae_with_error.cfg.use_error_term=True\n", + "model.add_sae(l5_sae_with_error)\n", + "print(\"Attached SAEs after adding l5_sae_with_error:\", model.acts_to_saes)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now the output of each attached SAE will be SAE(x) + error_term = x. We can sanity check this by confirming that running with SAEs produces the same logits without SAEs." + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "logits_with_saes = model(tokens)\n", + "logit_diff_with_saes = logits_to_ave_logit_diff(logits_with_saes, answer_tokens)\n", + "\n", + "assert torch.allclose(logits_with_saes, original_logits, atol=1e-4)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we can compare ablations of each feature to ablating the error node. We'll start by ablating each feature on each prompt, and then the error nodes. We'll append the effects from ablating error nodes to the rightmost column on the heatmap:" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 141/141 [00:04<00:00, 32.33it/s]\n" + ] + }, + { + "data": { + "application/vnd.plotly.v1+json": { + "config": { + "plotlyServerURL": "https://plot.ly" + }, + "data": [ + { + "coloraxis": "coloraxis", + "hovertemplate": "Feature Idx: %{x}
Prompt Idx: %{y}
color: %{z}", + "name": "0", + "type": "heatmap", + "x": [ + "46", + "345", + "702", + "1372", + "1755", + "1965", + "2457", + "2496", + "2646", + "2999", + "3047", + "4569", + "5132", + "5203", + "5508", + "5940", + "6144", + "6371", + "6515", + "6558", + "6812", + "7092", + "7515", + "7907", + "8063", + "8623", + "8737", + "8768", + "9096", + "9102", + "9186", + "9463", + "9746", + "9913", + "10581", + "10894", + "12109", + "12485", + "12764", + "12866", + "13063", + "13624", + "13707", + "13777", + "14844", + "15050", + "15170", + "15696", + "16178", + "16892", + "17156", + "17259", + "17497", + "17854", + "18043", + "18210", + "18318", + "18385", + "18440", + "18920", + "19183", + "19263", + "19442", + "19524", + "19573", + "20838", + "21151", + "21657", + "22108", + "23578", + "24091", + "24217", + "25792", + "26373", + "26410", + "27535", + "27787", + "27811", + "27960", + "28061", + "28241", + "28242", + "28254", + "28349", + "28977", + "29027", + "29482", + "29603", + "29700", + "29822", + "32177", + "32920", + "33320", + "33730", + "33966", + "34177", + "34334", + "34947", + "35403", + "35425", + "35579", + "35665", + "35815", + "36109", + "36172", + "36451", + "36767", + "36917", + "38570", + "39962", + "40409", + "40418", + "40661", + "41162", + "41185", + "41552", + "42024", + "42161", + "42437", + "42577", + "42882", + "42931", + "43035", + "43414", + "43643", + "43662", + "44203", + "44256", + "44452", + "44652", + "45179", + "45814", + "45984", + "46880", + "47117", + "47170", + "47231", + "47313", + "47680", + "48063", + "48703", + "error" + ], + "xaxis": "x", + "yaxis": "y", + "z": [ + [ + 0.0012617111206054688, + -9.5367431640625e-7, + -9.5367431640625e-7, + 0.0016908645629882812, + -0.0002231597900390625, + -9.5367431640625e-7, + -9.5367431640625e-7, + -0.00029659271240234375, + -9.5367431640625e-7, + -0.03279590606689453, + -0.07254886627197266, + -9.5367431640625e-7, + 0.00013065338134765625, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -0.014922142028808594, + -0.0044403076171875, + 0.0007047653198242188, + -0.00428009033203125, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -0.039069175720214844, + -9.5367431640625e-7, + -9.5367431640625e-7, + -0.007334709167480469, + 0.00033092498779296875, + -0.0017004013061523438, + 0.0026845932006835938, + 0.00043010711669921875, + -0.11128997802734375, + -0.0038976669311523438, + -0.006033897399902344, + -9.5367431640625e-7, + -0.00027751922607421875, + -9.5367431640625e-7, + 0.0006570816040039062, + -0.0004291534423828125, + -9.5367431640625e-7, + -0.0035734176635742188, + -0.0033063888549804688, + -9.5367431640625e-7, + -9.5367431640625e-7, + 0.0033960342407226562, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -0.0030546188354492188, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -0.0000972747802734375, + -0.0001811981201171875, + -9.5367431640625e-7, + -0.004569053649902344, + -0.013583183288574219, + -9.5367431640625e-7, + 0.02047252655029297, + -0.02572154998779297, + -9.5367431640625e-7, + -0.0006608963012695312, + -9.5367431640625e-7, + -9.5367431640625e-7, + 0.02255725860595703, + -0.05519580841064453, + -0.0033473968505859375, + -0.0000057220458984375, + -0.0026073455810546875, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -0.02097320556640625, + 0.008440971374511719, + -9.5367431640625e-7, + -0.004597663879394531, + 0.00159454345703125, + 0.0001544952392578125, + 0.005199432373046875, + 0.0007762908935546875, + -9.5367431640625e-7, + -9.5367431640625e-7, + -0.0032625198364257812, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -0.015192985534667969, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -0.018138885498046875, + -9.5367431640625e-7, + 0.010298728942871094, + -9.5367431640625e-7, + -0.0031423568725585938, + -9.5367431640625e-7, + 0.004242897033691406, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -0.010041236877441406, + 0.0010347366333007812, + 0.006011962890625, + -9.5367431640625e-7, + -9.5367431640625e-7, + 0.00301361083984375, + -0.04584026336669922, + 0.0002079010009765625, + -9.5367431640625e-7, + -9.5367431640625e-7, + -0.0002574920654296875, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -0.45942211151123047, + -0.0008325576782226562, + 0.00041484832763671875, + -9.5367431640625e-7, + -0.023777008056640625, + 0.0000514984130859375, + -9.5367431640625e-7, + -9.5367431640625e-7, + -0.00030422210693359375, + 0.0006666183471679688, + -9.5367431640625e-7, + 0.004633903503417969, + -9.5367431640625e-7, + -0.008234977722167969, + -0.07327461242675781 + ], + [ + 0.000003814697265625, + 0.000003814697265625, + -0.00208282470703125, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + -0.0012912750244140625, + -0.01760101318359375, + 0.000003814697265625, + 0.057277679443359375, + 0.013429641723632812, + 0.000003814697265625, + 0.000003814697265625, + -0.0000457763671875, + -0.0027828216552734375, + -0.0055084228515625, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + -0.2744255065917969, + 0.000003814697265625, + 0.000003814697265625, + 0.0021514892578125, + 0.000003814697265625, + 0.06994247436523438, + 0.0048542022705078125, + 0.000003814697265625, + 0.000003814697265625, + -0.0567169189453125, + 0.000003814697265625, + 0.012315750122070312, + 0.0066585540771484375, + 0.07937240600585938, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.028867721557617188, + 0.000003814697265625, + 0.000003814697265625, + 0.0074901580810546875, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.009624481201171875, + 0.000003814697265625, + -0.009510040283203125, + 0.0032100677490234375, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.10918617248535156, + 0.000003814697265625, + 0.026102066040039062, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.000946044921875, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + -0.041675567626953125, + 0.000003814697265625, + 0.000003814697265625, + -0.0066776275634765625, + 0.03926849365234375, + 0.03615379333496094, + 0.027612686157226562, + 0.000003814697265625, + -0.0004673004150390625, + -0.1435985565185547, + -0.00030517578125, + 0.059326171875, + 0.000003814697265625, + 0.000003814697265625, + 0.020435333251953125, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + -0.11923980712890625, + -0.009393692016601562, + 0.000003814697265625, + 0.011783599853515625, + 0.06122589111328125, + 0.000003814697265625, + 0.000003814697265625, + 0.0002918243408203125, + 0.000003814697265625, + 0.000003814697265625, + 0.001491546630859375, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + -0.0050716400146484375, + 0.000003814697265625, + 0.000003814697265625, + 0.025064468383789062, + 0.000003814697265625, + -0.0467529296875, + 0.000003814697265625, + 0.000003814697265625, + 0.0014934539794921875, + 0.00043487548828125, + 0.028188705444335938, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.001995086669921875, + 0.000003814697265625, + 0.000003814697265625, + 0.13014602661132812, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.0005893707275390625, + 0.012182235717773438, + 0.000003814697265625, + 0.000003814697265625, + 0.11103057861328125, + 0.042850494384765625, + 0.030099868774414062, + 0.000003814697265625, + -0.0047321319580078125, + 0.0000133514404296875, + -0.0320587158203125, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.031030654907226562, + -0.002635955810546875 + ], + [ + 0.007018089294433594, + 0, + 0, + 0.0028057098388671875, + 0, + 0, + 0, + -0.010999679565429688, + 0, + -0.1419973373413086, + -0.24188613891601562, + 0, + 0.0003147125244140625, + 0, + 0, + 0, + 0.009432792663574219, + 0, + -0.000125885009765625, + 0, + 0, + 0.00017070770263671875, + 0.011651992797851562, + -0.00225830078125, + 0, + -0.0014581680297851562, + 0.00020122528076171875, + -0.030771255493164062, + -0.03744316101074219, + -0.034499168395996094, + -0.00374603271484375, + 0, + 0.0011348724365234375, + 0, + -0.0302276611328125, + -0.08229637145996094, + -0.00048160552978515625, + 0, + -0.00640869140625, + 0.0001277923583984375, + 0, + -0.0008974075317382812, + 0.00022983551025390625, + -0.2322559356689453, + -0.0050449371337890625, + -0.010677337646484375, + 0, + 0.014942169189453125, + 0, + 0.0008764266967773438, + 0.00417327880859375, + 0, + -0.015301704406738281, + 0, + -0.0008974075317382812, + -0.04426097869873047, + 0.005242347717285156, + 0, + 0, + 0, + -0.009447097778320312, + 0, + 0, + -0.0011806488037109375, + 0, + -0.0045909881591796875, + 0.015285491943359375, + -0.034976959228515625, + -0.013401985168457031, + 0, + 0.1357421875, + -0.09111690521240234, + 0, + 0.00013065338134765625, + 0.0002460479736328125, + 0, + 0.04656982421875, + -0.09346866607666016, + 0, + -0.005030632019042969, + 0.0001125335693359375, + 0, + 0, + 0, + -0.07491683959960938, + 0.006598472595214844, + 0, + -0.014060020446777344, + -0.008306503295898438, + -0.0054874420166015625, + -0.0004930496215820312, + 0, + 0, + 0, + -0.008953094482421875, + 0, + 0, + 0, + -0.03713417053222656, + 0, + 0, + 0, + 0, + -0.028200149536132812, + 0, + 0.036255836486816406, + 0, + -0.03178215026855469, + 0, + -0.012192726135253906, + -0.002147674560546875, + -0.0005474090576171875, + -0.0021409988403320312, + -0.030725479125976562, + 0, + 0.0008029937744140625, + 0, + 0, + 0, + -0.29135894775390625, + 0, + 0, + 0, + 0, + 0, + -0.0027914047241210938, + -0.00022125244140625, + -0.8653240203857422, + 0, + -0.05593109130859375, + 0, + -0.04123210906982422, + 0, + 0, + 0.015351295471191406, + 0, + 0, + 0, + 0.018423080444335938, + -0.0000476837158203125, + -0.0023584365844726562, + -0.3282146453857422 + ], + [ + 9.5367431640625e-7, + 9.5367431640625e-7, + 0.0001983642578125, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + -0.010341644287109375, + 0.07198715209960938, + 0.14725303649902344, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 0.0002918243408203125, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 0.011704444885253906, + 9.5367431640625e-7, + 9.5367431640625e-7, + -0.3150959014892578, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + -0.039947509765625, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 0.15607547760009766, + 9.5367431640625e-7, + 0.09917640686035156, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 0.019521713256835938, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 0.012205123901367188, + 9.5367431640625e-7, + -0.0005893707275390625, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 0.07062149047851562, + 0.000492095947265625, + 0.014776229858398438, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 0.0557098388671875, + 0.15409469604492188, + 9.5367431640625e-7, + 9.5367431640625e-7, + -0.0007076263427734375, + -0.24256324768066406, + 9.5367431640625e-7, + 0.0858917236328125, + 9.5367431640625e-7, + 9.5367431640625e-7, + 0.007343292236328125, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + -0.11646080017089844, + 9.5367431640625e-7, + 9.5367431640625e-7, + 0.05528736114501953, + 0.0847921371459961, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 0.00428009033203125, + -0.0056171417236328125, + 9.5367431640625e-7, + 9.5367431640625e-7, + 0.0066967010498046875, + 9.5367431640625e-7, + -0.006005287170410156, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 0.01735687255859375, + -0.0037336349487304688, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 0.09533309936523438, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 0.009324073791503906, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 0.007989883422851562, + 9.5367431640625e-7, + 0.0064525604248046875, + 9.5367431640625e-7, + -0.06574440002441406, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 0.5591859817504883 + ], + [ + -0.0009012222290039062, + 0.00001239776611328125, + 0.00001239776611328125, + -0.0006313323974609375, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + -0.000461578369140625, + 0.00001239776611328125, + -0.055993080139160156, + -0.24974536895751953, + 0.00001239776611328125, + 0.0011262893676757812, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + -0.0025796890258789062, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + -0.030013084411621094, + -0.012925148010253906, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + -0.0253448486328125, + 0.00001239776611328125, + 0.0012464523315429688, + 0.021536827087402344, + 0.00001239776611328125, + 0.00001239776611328125, + -0.00009822845458984375, + 0.00001239776611328125, + -0.09924793243408203, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.006188392639160156, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + -0.0010576248168945312, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.0008172988891601562, + 0.00001239776611328125, + 0.00001239776611328125, + -0.0020704269409179688, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + -0.09985160827636719, + 0.00001239776611328125, + 0.00001239776611328125, + 0.036945343017578125, + 0.025011062622070312, + 0.00001239776611328125, + 0.004599571228027344, + 0.00001239776611328125, + 0.00001239776611328125, + 0.027939796447753906, + -0.07974910736083984, + 0.00001239776611328125, + 0.00001239776611328125, + -0.00038242340087890625, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + -0.035175323486328125, + 0.00001239776611328125, + 0.00001239776611328125, + -0.0047245025634765625, + -0.008166313171386719, + -0.008578300476074219, + 0.0018529891967773438, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + -0.0016679763793945312, + 0.00001239776611328125, + 0.00001239776611328125, + -0.0028676986694335938, + -0.04880046844482422, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + -0.0053462982177734375, + 0.00001239776611328125, + 0.1658468246459961, + 0.00001239776611328125, + -0.0024824142456054688, + 0.00001239776611328125, + 0.025139808654785156, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + -0.027915000915527344, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + -0.14544200897216797, + 0.00001239776611328125, + 0.020270347595214844, + 0.007473945617675781, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + -0.8424196243286133, + 0.00001239776611328125, + -0.007409095764160156, + -0.00318145751953125, + -0.015982627868652344, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00034046173095703125, + 0.10727787017822266, + 0.00001239776611328125, + 0.00001239776611328125, + -0.5388059616088867 + ], + [ + -0.00000762939453125, + -0.00000762939453125, + 0.0019397735595703125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.022940635681152344, + 0.07428932189941406, + 0.29994869232177734, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.016974449157714844, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.4772310256958008, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + 0.05463600158691406, + -0.00000762939453125, + -0.00000762939453125, + 0.004734992980957031, + -0.12352275848388672, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + 0.15236186981201172, + -0.00000762939453125, + 0.27855396270751953, + -0.00000762939453125, + -0.00000762939453125, + 0.001430511474609375, + -0.00000762939453125, + -0.00000762939453125, + -0.016387939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + 0.0008668899536132812, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + 0.06814861297607422, + -0.00000762939453125, + 0.00351715087890625, + -0.00000762939453125, + 0.0061588287353515625, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + 0.02480602264404297, + 0.31668567657470703, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.4413900375366211, + -0.00000762939453125, + 0.1517791748046875, + -0.00000762939453125, + -0.00000762939453125, + 0.010898590087890625, + -0.00000762939453125, + 0.006583213806152344, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.04946422576904297, + -0.00000762939453125, + -0.00000762939453125, + 0.040429115295410156, + 0.1020956039428711, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.0008649826049804688, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.0054836273193359375, + -0.010519981384277344, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + 0.004588127136230469, + -0.006558418273925781, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + 0.07750797271728516, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + 0.03235149383544922, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + 0.02773571014404297, + 0.08978557586669922, + -0.00000762939453125, + 0.008780479431152344, + -0.00000762939453125, + -0.0327301025390625, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + 0.035370826721191406, + 0.19881343841552734 + ], + [ + 0.0041904449462890625, + -0.0000057220458984375, + -0.0000057220458984375, + 0.008458137512207031, + -0.0000057220458984375, + -0.0042858123779296875, + -0.0000057220458984375, + 0.002468109130859375, + -0.0000057220458984375, + -0.03716564178466797, + -0.10456657409667969, + -0.0047702789306640625, + -0.0000057220458984375, + -0.0000057220458984375, + -0.0000057220458984375, + 0.017671585083007812, + 0.0004062652587890625, + -0.0000057220458984375, + -0.0000057220458984375, + -0.0000057220458984375, + -0.0000057220458984375, + -0.00655364990234375, + 0.001873016357421875, + -0.0000057220458984375, + -0.0000057220458984375, + -0.0000057220458984375, + -0.0000057220458984375, + -0.04653644561767578, + -0.01836395263671875, + 0.014448165893554688, + -0.0209197998046875, + -0.0000057220458984375, + -0.0000057220458984375, + -0.0000057220458984375, + -0.006618499755859375, + 0.02408599853515625, + -0.0000057220458984375, + 0.0012884140014648438, + -0.0000057220458984375, + -0.0000057220458984375, + -0.0000057220458984375, + -0.0050144195556640625, + -0.0000057220458984375, + 0.036708831787109375, + -0.0000057220458984375, + -0.0056591033935546875, + -0.0004215240478515625, + 0.0014057159423828125, + -0.0000057220458984375, + -0.0000057220458984375, + -0.0000057220458984375, + -0.0000057220458984375, + -0.033232688903808594, + -0.0000057220458984375, + -0.0000057220458984375, + -0.008130073547363281, + 0.016930580139160156, + -0.0000057220458984375, + -0.0012025833129882812, + -0.0000057220458984375, + 0.000545501708984375, + -0.0000057220458984375, + -0.0004673004150390625, + -0.0000057220458984375, + 0.0038089752197265625, + -0.008646011352539062, + -0.0000057220458984375, + -0.008909225463867188, + -0.011255264282226562, + -0.0000057220458984375, + 0.0925750732421875, + -0.0064563751220703125, + -0.0000057220458984375, + 0.0011615753173828125, + 0.00002956390380859375, + -0.0000057220458984375, + 0.07063961029052734, + -0.030902862548828125, + -0.0000057220458984375, + -0.0010814666748046875, + 0.00038909912109375, + -0.0000057220458984375, + -0.0000057220458984375, + -0.0013380050659179688, + -0.022397994995117188, + 0.027740478515625, + -0.0000057220458984375, + -0.0000057220458984375, + 0.01797771453857422, + 0.009552955627441406, + 0.01857471466064453, + -0.0000057220458984375, + -0.0000057220458984375, + -0.0000057220458984375, + -0.0000057220458984375, + -0.0000057220458984375, + -0.0000057220458984375, + -0.0000057220458984375, + -0.0055103302001953125, + -0.0000057220458984375, + -0.0000057220458984375, + -0.01697063446044922, + -0.0159149169921875, + -0.011240959167480469, + 0.000301361083984375, + 0.020501136779785156, + -0.0000057220458984375, + -0.0006427764892578125, + -0.0000057220458984375, + 0.04800224304199219, + -0.0000057220458984375, + -0.0000057220458984375, + -0.0000057220458984375, + -0.00001811981201171875, + 0.0005950927734375, + 0.00732421875, + -0.0000057220458984375, + 0.001216888427734375, + 0.00897216796875, + -0.1255035400390625, + 0.001003265380859375, + 0.006274223327636719, + 0.0026502609252929688, + -0.00449371337890625, + -0.0023517608642578125, + -0.0000057220458984375, + -0.0000057220458984375, + -0.6521244049072266, + -0.009072303771972656, + 0.013387680053710938, + -0.0000057220458984375, + -0.022745132446289062, + -0.0000057220458984375, + -0.0000057220458984375, + 0.000606536865234375, + -0.0011501312255859375, + -0.0000057220458984375, + -0.0000057220458984375, + 0.023046493530273438, + -0.0000057220458984375, + -0.008263587951660156, + -0.11597061157226562 + ], + [ + -0.0037221908569335938, + 0.00225830078125, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.05941295623779297, + 0.04140281677246094, + 0.24284648895263672, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.11462688446044922, + 0.012240409851074219, + 0.0012884140014648438, + -0.00001049041748046875, + 0.01781749725341797, + 0.005211830139160156, + -0.0016298294067382812, + -0.2994966506958008, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + 0.26962947845458984, + -0.00001049041748046875, + 0.050202369689941406, + 0.04053211212158203, + -0.30355358123779297, + -0.00001049041748046875, + -0.0013666152954101562, + -0.00001049041748046875, + 0.06442928314208984, + -0.00001049041748046875, + 0.04406547546386719, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00763702392578125, + -0.00001049041748046875, + -0.03402233123779297, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.005751609802246094, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + 0.13191986083984375, + -0.00001049041748046875, + -0.031653404235839844, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.02394390106201172, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.1073293685913086, + 0.20270729064941406, + 0.02746295928955078, + -0.00001049041748046875, + 0.020377159118652344, + -0.31055259704589844, + -0.043480873107910156, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + 0.04507160186767578, + -0.0014734268188476562, + -0.00001049041748046875, + 0.048813819885253906, + -0.00001049041748046875, + -0.00001049041748046875, + -0.1407604217529297, + -0.00001049041748046875, + -0.00001049041748046875, + 0.013548851013183594, + 0.016210556030273438, + -0.00001049041748046875, + -0.011261940002441406, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00029277801513671875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + 0.008993148803710938, + -0.020813941955566406, + -0.00001049041748046875, + -0.00001049041748046875, + -0.008435249328613281, + -0.021961212158203125, + -0.04410362243652344, + 0.1307668685913086, + 0.005297660827636719, + -0.00001049041748046875, + 0.006031990051269531, + 0.016150474548339844, + -0.00001049041748046875, + -0.00001049041748046875, + 0.01802349090576172, + 0.0018205642700195312, + 0.0016574859619140625, + 0.0005712509155273438, + -0.00001049041748046875, + -0.00001049041748046875, + 0.02598094940185547, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + 0.02737903594970703, + 0.039580345153808594, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + 0.09876728057861328, + 0.035803794860839844, + -0.00001049041748046875, + -0.027251243591308594, + -0.00001049041748046875, + -0.07061004638671875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + 0.08719158172607422, + -0.37606334686279297 + ] + ] + } + ], + "layout": { + "coloraxis": { + "cmid": 0, + "colorscale": [ + [ + 0, + "rgb(103,0,31)" + ], + [ + 0.1, + "rgb(178,24,43)" + ], + [ + 0.2, + "rgb(214,96,77)" + ], + [ + 0.3, + "rgb(244,165,130)" + ], + [ + 0.4, + "rgb(253,219,199)" + ], + [ + 0.5, + "rgb(247,247,247)" + ], + [ + 0.6, + "rgb(209,229,240)" + ], + [ + 0.7, + "rgb(146,197,222)" + ], + [ + 0.8, + "rgb(67,147,195)" + ], + [ + 0.9, + "rgb(33,102,172)" + ], + [ + 1, + "rgb(5,48,97)" + ] + ] + }, + "template": { + "data": { + "bar": [ + { + "error_x": { + "color": "#2a3f5f" + }, + "error_y": { + "color": "#2a3f5f" + }, + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "bar" + } + ], + "barpolar": [ + { + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "barpolar" + } + ], + "carpet": [ + { + "aaxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "baxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "type": "carpet" + } + ], + "choropleth": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "choropleth" + } + ], + "contour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "contour" + } + ], + "contourcarpet": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "contourcarpet" + } + ], + "heatmap": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmap" + } + ], + "heatmapgl": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmapgl" + } + ], + "histogram": [ + { + "marker": { + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "histogram" + } + ], + "histogram2d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2d" + } + ], + "histogram2dcontour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2dcontour" + } + ], + "mesh3d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "mesh3d" + } + ], + "parcoords": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "parcoords" + } + ], + "pie": [ + { + "automargin": true, + "type": "pie" + } + ], + "scatter": [ + { + "fillpattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + }, + "type": "scatter" + } + ], + "scatter3d": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatter3d" + } + ], + "scattercarpet": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattercarpet" + } + ], + "scattergeo": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergeo" + } + ], + "scattergl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergl" + } + ], + "scattermapbox": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermapbox" + } + ], + "scatterpolar": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolar" + } + ], + "scatterpolargl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolargl" + } + ], + "scatterternary": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterternary" + } + ], + "surface": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "surface" + } + ], + "table": [ + { + "cells": { + "fill": { + "color": "#EBF0F8" + }, + "line": { + "color": "white" + } + }, + "header": { + "fill": { + "color": "#C8D4E3" + }, + "line": { + "color": "white" + } + }, + "type": "table" + } + ] + }, + "layout": { + "annotationdefaults": { + "arrowcolor": "#2a3f5f", + "arrowhead": 0, + "arrowwidth": 1 + }, + "autotypenumbers": "strict", + "coloraxis": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "colorscale": { + "diverging": [ + [ + 0, + "#8e0152" + ], + [ + 0.1, + "#c51b7d" + ], + [ + 0.2, + "#de77ae" + ], + [ + 0.3, + "#f1b6da" + ], + [ + 0.4, + "#fde0ef" + ], + [ + 0.5, + "#f7f7f7" + ], + [ + 0.6, + "#e6f5d0" + ], + [ + 0.7, + "#b8e186" + ], + [ + 0.8, + "#7fbc41" + ], + [ + 0.9, + "#4d9221" + ], + [ + 1, + "#276419" + ] + ], + "sequential": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "sequentialminus": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ] + }, + "colorway": [ + "#636efa", + "#EF553B", + "#00cc96", + "#ab63fa", + "#FFA15A", + "#19d3f3", + "#FF6692", + "#B6E880", + "#FF97FF", + "#FECB52" + ], + "font": { + "color": "#2a3f5f" + }, + "geo": { + "bgcolor": "white", + "lakecolor": "white", + "landcolor": "#E5ECF6", + "showlakes": true, + "showland": true, + "subunitcolor": "white" + }, + "hoverlabel": { + "align": "left" + }, + "hovermode": "closest", + "mapbox": { + "style": "light" + }, + "paper_bgcolor": "white", + "plot_bgcolor": "#E5ECF6", + "polar": { + "angularaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "radialaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "scene": { + "xaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "yaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "zaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + } + }, + "shapedefaults": { + "line": { + "color": "#2a3f5f" + } + }, + "ternary": { + "aaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "baxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "caxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "title": { + "x": 0.05 + }, + "xaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + }, + "yaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + } + } + }, + "title": { + "text": "Change in logit diff when ablating L5 SAE features for all prompts at pos 10" + }, + "xaxis": { + "anchor": "y", + "constrain": "domain", + "domain": [ + 0, + 1 + ], + "scaleanchor": "y", + "title": { + "text": "Feature Idx" + } + }, + "yaxis": { + "anchor": "x", + "autorange": "reversed", + "constrain": "domain", + "domain": [ + 0, + 1 + ], + "title": { + "text": "Prompt Idx" + } + } + } + }, + "text/html": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def ablate_sae_feature(sae_acts, hook, pos, feature_id):\n", + " if pos is None:\n", + " sae_acts[:, :, feature_id] = 0.\n", + " else:\n", + " sae_acts[:, pos, feature_id] = 0.\n", + " return sae_acts\n", + "\n", + "layer = 5\n", + "hooked_encoder = model.acts_to_saes[utils.get_act_name('z', layer)]\n", + "all_live_features = torch.arange(hooked_encoder.cfg.d_sae)[live_feature_union.cpu()]\n", + "\n", + "causal_effects = torch.zeros((len(prompts), all_live_features.shape[0]))\n", + "fid_to_idx = {fid.item(): idx for idx, fid in enumerate(all_live_features)}\n", + "\n", + "\n", + "abl_layer, abl_pos = 5, 10\n", + "for feature_id in tqdm.tqdm(all_live_features):\n", + " feature_id = feature_id.item()\n", + " abl_feature_logits = model.run_with_hooks(\n", + " tokens,\n", + " return_type=\"logits\",\n", + " fwd_hooks=[(utils.get_act_name('z', abl_layer) + \".hook_sae_acts_post\", partial(ablate_sae_feature, pos=abl_pos, feature_id=feature_id))]\n", + " ) # [batch, seq, vocab]\n", + " \n", + " abl_feature_logit_diff = logits_to_ave_logit_diff(abl_feature_logits, answer_tokens, per_prompt=True) # [batch]\n", + " causal_effects[:, fid_to_idx[feature_id]] = abl_feature_logit_diff - original_per_prompt_logit_diff\n", + "\n", + "def able_sae_error(sae_error, hook, pos):\n", + " if pos is None:\n", + " sae_error = 0.\n", + " else:\n", + " sae_error[:, pos, ...] = 0.\n", + " return sae_error\n", + "\n", + "\n", + "abl_error_logits = model.run_with_hooks(\n", + " tokens,\n", + " return_type=\"logits\",\n", + " fwd_hooks=[(utils.get_act_name('z', abl_layer) + \".hook_sae_error\", partial(able_sae_error, pos=abl_pos))]\n", + ") # [batch, seq, vocab]\n", + "\n", + "abl_error_logit_diff = logits_to_ave_logit_diff(abl_error_logits, answer_tokens, per_prompt=True) # [batch]\n", + "error_abl_effect = abl_error_logit_diff - original_per_prompt_logit_diff\n", + "\n", + "\n", + "causal_effects_with_error = torch.cat([causal_effects, error_abl_effect.unsqueeze(-1).cpu()], dim=-1)\n", + "imshow(causal_effects_with_error, title=f\"Change in logit diff when ablating L{abl_layer} SAE features for all prompts at pos {abl_pos}\", xaxis=\"Feature Idx\", yaxis=\"Prompt Idx\", x=list(map(str, all_live_features.tolist()))+[\"error\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can see that on some prompts, ablating the error term (right most column) does have a non trivial effect on the logit diff, although I don't see a clear pattern. It seems useful to include this term when doing causal interventions to get a better sense of how much the SAE features are actually explaining. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Attribution patching " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "Both [Anthropic](https://transformer-circuits.pub/2024/march-update/index.html#feature-heads) and [Marks et al](https://arxiv.org/abs/2403.19647v2). also demonstrated the use of gradient based attribution techniques as a substitute for activation patching on SAE features. The key idea is that patching / ablations (as we did above) can be slow, as it requires a new forward pass for each patch. This seems especially problematic when dealing with SAEs with tens of thousands of features per activation. They find that gradient based attribution techniques like [attribution patching](https://www.neelnanda.io/mechanistic-interpretability/attribution-patching) are good approximations, allowing for more efficient and scalable circuit analysis with SAEs.\n", + "\n", + "With `HookedSAETransformer`, added SAEs are automatically spliced into the computational graph, allowing us to implement this easily. Let's implement attribution patching for every L5 SAE feature to find causally relevant SAE features with just one forward and one backward pass." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.set_grad_enabled(True)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor(-7.6294e-06, device='cuda:0', grad_fn=)\n", + "Clean Value: -7.62939453125e-06\n", + "Clean Activations Cached: 1\n", + "Clean Gradients Cached: 1\n" + ] + } + ], + "source": [ + "from transformer_lens import ActivationCache\n", + "filter_sae_acts = lambda name: (\"hook_sae_acts_post\" in name)\n", + "def get_cache_fwd_and_bwd(model, tokens, metric):\n", + " model.reset_hooks()\n", + " cache = {}\n", + " def forward_cache_hook(act, hook):\n", + " cache[hook.name] = act.detach()\n", + " model.add_hook(filter_sae_acts, forward_cache_hook, \"fwd\")\n", + "\n", + " grad_cache = {}\n", + " def backward_cache_hook(act, hook):\n", + " grad_cache[hook.name] = act.detach()\n", + " model.add_hook(filter_sae_acts, backward_cache_hook, \"bwd\")\n", + "\n", + " value = metric(model(tokens))\n", + " print(value)\n", + " value.backward()\n", + " model.reset_hooks()\n", + " return value.item(), ActivationCache(cache, model), ActivationCache(grad_cache, model)\n", + "\n", + "\n", + "BASELINE = original_per_prompt_logit_diff\n", + "def ioi_metric(logits, answer_tokens=answer_tokens):\n", + " return (logits_to_ave_logit_diff(logits, answer_tokens, per_prompt=True) - BASELINE).sum()\n", + "\n", + "clean_tokens = tokens.clone()\n", + "clean_value, clean_cache, clean_grad_cache = get_cache_fwd_and_bwd(model, clean_tokens, ioi_metric)\n", + "print(\"Clean Value:\", clean_value)\n", + "print(\"Clean Activations Cached:\", len(clean_cache))\n", + "print(\"Clean Gradients Cached:\", len(clean_grad_cache))" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.plotly.v1+json": { + "config": { + "plotlyServerURL": "https://plot.ly" + }, + "data": [ + { + "coloraxis": "coloraxis", + "hovertemplate": "Feature Idx: %{x}
Prompt Idx: %{y}
color: %{z}", + "name": "0", + "type": "heatmap", + "x": [ + "46", + "345", + "702", + "1372", + "1755", + "1965", + "2457", + "2496", + "2646", + "2999", + "3047", + "4569", + "5132", + "5203", + "5508", + "5940", + "6144", + "6371", + "6515", + "6558", + "6812", + "7092", + "7515", + "7907", + "8063", + "8623", + "8737", + "8768", + "9096", + "9102", + "9186", + "9463", + "9746", + "9913", + "10581", + "10894", + "12109", + "12485", + "12764", + "12866", + "13063", + "13624", + "13707", + "13777", + "14844", + "15050", + "15170", + "15696", + "16178", + "16892", + "17156", + "17259", + "17497", + "17854", + "18043", + "18210", + "18318", + "18385", + "18440", + "18920", + "19183", + "19263", + "19442", + "19524", + "19573", + "20838", + "21151", + "21657", + "22108", + "23578", + "24091", + "24217", + "25792", + "26373", + "26410", + "27535", + "27787", + "27811", + "27960", + "28061", + "28241", + "28242", + "28254", + "28349", + "28977", + "29027", + "29482", + "29603", + "29700", + "29822", + "32177", + "32920", + "33320", + "33730", + "33966", + "34177", + "34334", + "34947", + "35403", + "35425", + "35579", + "35665", + "35815", + "36109", + "36172", + "36451", + "36767", + "36917", + "38570", + "39962", + "40409", + "40418", + "40661", + "41162", + "41185", + "41552", + "42024", + "42161", + "42437", + "42577", + "42882", + "42931", + "43035", + "43414", + "43643", + "43662", + "44203", + "44256", + "44452", + "44652", + "45179", + "45814", + "45984", + "46880", + "47117", + "47170", + "47231", + "47313", + "47680", + "48063", + "48703" + ], + "xaxis": "x", + "yaxis": "y", + "z": [ + [ + 0.001567811705172062, + 0, + 0, + 0.001697835512459278, + 0.00011560246639419347, + 0, + 0, + -0.0002851475146599114, + 0, + -0.030827227979898453, + -0.06409652531147003, + 0, + 0.00015167289529927075, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + -0.013627146370708942, + -0.004393726587295532, + 0.0015328703448176384, + -0.0038613511715084314, + 0, + 0, + 0, + 0, + -0.02049136720597744, + 0, + 0, + -0.007114107254892588, + 0.0003477374848444015, + -0.001384311355650425, + 0.003183899214491248, + 0.0004558839718811214, + -0.059277813881635666, + -0.0035793157294392586, + -0.00589390005916357, + 0, + -0.0001910730206873268, + 0, + 0.0006608504336327314, + -0.0004212319909129292, + 0, + -0.003545185085386038, + -0.00327106611803174, + 0, + 0, + 0.0040074847638607025, + 0, + 0, + 0, + -0.0026069351006299257, + 0, + 0, + 0, + -0.00008433026232523844, + -0.00018646706303115934, + 0, + -0.00439279293641448, + -0.013254894874989986, + 0, + 0.050094299018383026, + -0.021308520808815956, + 0, + -0.0006410681526176631, + 0, + 0, + 0.02329532988369465, + -0.05166983604431152, + -0.002982117934152484, + -0.000014124364497547504, + -0.0020334068685770035, + 0, + 0, + 0, + -0.02020590752363205, + 0.00998645182698965, + 0, + -0.004585121292620897, + 0.005916096270084381, + 0.0018219061894342303, + 0.005700498353689909, + 0.0008085825829766691, + 0, + 0, + -0.0032405084930360317, + 0, + 0, + 0, + -0.014961971901357174, + 0, + 0, + 0, + 0, + -0.016915086656808853, + 0, + 0.016825370490550995, + 0, + -0.00311169121414423, + 0, + 0.005266942549496889, + 0, + 0, + 0, + -0.009660078212618828, + 0.0010975055629387498, + 0.006078756880015135, + 0, + 0, + 0.003166533075273037, + -0.044512320309877396, + 0.0002630578528624028, + 0, + 0, + -0.00025422731414437294, + 0, + 0, + 0, + -0.3718416392803192, + -0.0008081833366304636, + 0.00043700754758901894, + 0, + -0.023154418915510178, + 0.00004691413778346032, + 0, + 0, + -0.0002914638607762754, + 0.0006733346963301301, + 0, + 0.008972969837486744, + 0, + -0.008168808184564114 + ], + [ + 0, + 0, + -0.0006953283445909619, + 0, + 0, + 0, + -0.001286927843466401, + -0.017273705452680588, + 0, + 0.05898163467645645, + 0.013462062925100327, + 0, + 0, + -0.00003325308352941647, + -0.0027551515959203243, + -0.004652985371649265, + 0, + 0, + 0, + 0, + 0, + 0, + -0.21421866118907928, + 0, + 0, + 0.002191215055063367, + 0, + 0.07645706832408905, + 0.0052618952468037605, + 0, + 0, + -0.020269982516765594, + 0, + 0.013446477241814137, + 0.0068704248405992985, + 0.08710267394781113, + 0, + 0, + 0, + 0, + 0.028982989490032196, + 0, + 0, + 0.014961526729166508, + 0, + 0, + 0, + 0, + 0.011233230121433735, + 0, + -0.009112805128097534, + 0.003226917004212737, + 0, + 0, + 0, + 0.112985759973526, + 0, + 0.028253009542822838, + 0, + 0, + 0, + 0.0009787877788767219, + 0, + 0, + 0, + 0, + -0.03986968472599983, + 0, + 0, + -0.006135094445198774, + 0.04977395758032799, + 0.0397123359143734, + 0.027974072843790054, + 0, + -0.00044811973930336535, + -0.10083132237195969, + 0.000008234118467953522, + 0.06165996566414833, + 0, + 0, + 0.021058127284049988, + 0, + 0, + 0, + 0, + 0, + -0.08074336498975754, + -0.009298793971538544, + 0, + 0.012482613325119019, + 0.06513619422912598, + 0, + 0, + 0.00029019018984399736, + 0, + 0, + 0.0014882637187838554, + 0, + 0, + 0, + -0.004803473129868507, + 0, + 0, + 0.025678949430584908, + 0, + -0.04240157827734947, + 0, + 0, + 0.0015190609265118837, + 0.0006482255994342268, + 0.03654245659708977, + 0, + 0, + 0, + 0, + 0, + 0.0020186977926641703, + 0, + 0, + 0.17831696569919586, + 0, + 0, + 0, + 0.0005887048901058733, + 0.012331255711615086, + 0, + 0, + 0.11619613319635391, + 0.04687207192182541, + 0.03033648431301117, + 0, + -0.004195880610495806, + 0.00006391256465576589, + -0.03162289038300514, + 0, + 0, + 0, + 0, + 0, + 0, + 0.03672636300325394 + ], + [ + 0.00788492988795042, + 0, + 0, + 0.003685369621962309, + 0, + 0, + 0, + -0.010384900495409966, + 0, + -0.1327948272228241, + -0.22788244485855103, + 0, + 0.0003893508983310312, + 0, + 0, + 0, + 0.009530982933938503, + 0, + -0.0001286355109186843, + 0, + 0, + 0.0001596187794348225, + 0.011789986863732338, + -0.0022452236153185368, + 0, + -0.0014552043285220861, + 0.0002036036894423887, + -0.03003234602510929, + -0.036742936819791794, + -0.028862446546554565, + -0.003727517556399107, + 0, + 0.0011460097739472985, + 0, + -0.027142589911818504, + -0.054151974618434906, + -0.0004727205669041723, + 0, + -0.006094601005315781, + 0.00013960858632344753, + 0, + -0.0003665595140773803, + 0.00028091753483749926, + -0.17846877872943878, + -0.004990901332348585, + -0.010615025646984577, + 0, + 0.015916047617793083, + 0, + 0.0008773574372753501, + 0.004459311719983816, + 0, + -0.015235064551234245, + 0, + -0.0008741968194954097, + -0.04074608162045479, + 0.007227533031255007, + 0, + 0, + 0, + -0.007763775996863842, + 0, + 0, + -0.0011336231837049127, + 0, + -0.004542750306427479, + 0.016146792098879814, + -0.032868705689907074, + -0.013282506726682186, + 0, + 0.1884474903345108, + -0.07819699496030807, + 0, + 0.00013099861098453403, + 0.00024322106037288904, + 0, + 0.04764547944068909, + -0.09056885540485382, + 0, + -0.005007788073271513, + 0.000487087934743613, + 0, + 0, + 0, + -0.07196655869483948, + 0.007451012264937162, + 0, + -0.013892672955989838, + -0.005596193019300699, + -0.005349555052816868, + -0.00015437132969964296, + 0, + 0, + 0, + -0.00894666463136673, + 0, + 0, + 0, + -0.036862581968307495, + 0, + 0, + 0, + 0, + -0.026162482798099518, + 0, + 0.046491872519254684, + 0, + -0.030160455033183098, + 0, + -0.009029642678797245, + -0.0021479984279721975, + -0.0005375721957534552, + -0.002135993679985404, + -0.027962258085608482, + 0, + 0.0008057129452936351, + 0, + 0, + 0, + -0.26795026659965515, + 0, + 0, + 0, + 0, + 0, + -0.0027670287527143955, + -0.0002252299600513652, + -0.7548060417175293, + 0, + -0.05009680241346359, + 0, + -0.03914204612374306, + 0, + 0, + 0.016279445961117744, + 0, + 0, + 0, + 0.025662390515208244, + -0.000049459828005637974, + -0.0023572721984237432 + ], + [ + 0, + 0, + 0.0009027881897054613, + 0, + 0, + 0, + 0, + -0.01007400918751955, + 0.07334298640489578, + 0.15174342691898346, + 0, + 0, + 0, + 0, + 0, + 0.0007311829249374568, + 0, + 0, + 0, + 0.011839455924928188, + 0, + 0, + -0.2282165139913559, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + -0.017542533576488495, + 0, + 0, + 0, + 0.1636323779821396, + 0, + 0.10289037227630615, + 0, + 0, + 0, + 0, + 0, + 0.024433566257357597, + 0, + 0, + 0, + 0, + 0.013018166646361351, + 0, + -0.0005916667287237942, + 0, + 0, + 0, + 0, + 0.07111621648073196, + 0.0004984873230569065, + 0.015917964279651642, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.06262800097465515, + 0.17253385484218597, + 0, + 0, + -0.0007970984443090856, + -0.1451263427734375, + 0, + 0.08718064427375793, + 0, + 0, + 0.007446629460901022, + 0, + 0, + 0, + 0, + 0, + -0.09546831995248795, + 0, + 0, + 0.06110787391662598, + 0.08931172639131546, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.005256101489067078, + -0.00553735950961709, + 0, + 0, + 0.006732907146215439, + 0, + -0.005547903478145599, + 0, + 0, + 0, + 0.01766844280064106, + -0.0034187675919383764, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.1122211441397667, + 0, + 0, + 0, + 0.009442206472158432, + 0, + 0, + 0, + 0, + 0, + 0.00800288561731577, + 0, + 0.006613056641072035, + 0, + -0.06462590396404266, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + -0.0009047402418218553, + 0, + 0, + -0.0005877931835129857, + 0, + 0, + 0, + -0.0004729636711999774, + 0, + -0.05036322772502899, + -0.24687804281711578, + 0, + 0.001115482416935265, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + -0.0024291854351758957, + 0, + 0, + 0, + -0.029154174029827118, + -0.011211197823286057, + 0, + 0, + 0, + 0, + 0, + -0.0075091151520609856, + 0, + 0.0037634933833032846, + 0.022711526602506638, + 0, + 0, + -0.00011145337339257821, + 0, + -0.08350298553705215, + 0, + 0, + 0, + 0, + 0, + 0.0063380529172718525, + 0, + 0, + 0, + 0, + 0, + 0, + -0.0010615212377160788, + 0, + 0, + 0, + 0.001314864493906498, + 0, + 0, + -0.0020079570822417736, + 0, + 0, + 0, + -0.095857173204422, + 0, + 0, + 0.04977884888648987, + 0.04924672096967697, + 0, + 0.00675918348133564, + 0, + 0, + 0.02823697216808796, + -0.07869893312454224, + 0, + 0, + -0.00039145027403719723, + 0, + 0, + 0, + -0.03502006456255913, + 0, + 0, + -0.004709419794380665, + -0.007543480955064297, + -0.007213911972939968, + 0.0026987697929143906, + 0, + 0, + 0, + -0.0016787010245025158, + 0, + 0, + -0.002866228111088276, + -0.04759479686617851, + 0, + 0, + 0, + 0, + -0.005348640959709883, + 0, + 0.17661413550376892, + 0, + -0.0024743194226175547, + 0, + 0.0269751138985157, + 0, + 0, + 0, + -0.025461290031671524, + 0, + 0, + 0, + 0, + 0, + -0.14607883989810944, + 0, + 0.020490022376179695, + 0.007573024369776249, + 0, + 0, + 0, + 0, + -0.8939738869667053, + 0, + -0.006900197826325893, + -0.0031849159859120846, + -0.015817783772945404, + 0, + 0, + 0, + 0, + 0, + 0.00032859406201168895, + 0.11629504710435867, + 0, + 0 + ], + [ + 0, + 0, + 0.0020032059401273727, + 0, + 0, + 0, + 0, + -0.02256190776824951, + 0.07616151124238968, + 0.3106333911418915, + 0, + 0, + 0, + 0, + 0, + -0.014044971205294132, + 0, + 0, + 0, + 0, + 0, + 0, + -0.3483165502548218, + 0, + 0, + 0, + 0, + 0.05930393189191818, + 0, + 0, + 0.004992437083274126, + -0.08404884487390518, + 0, + 0, + 0, + 0.16281214356422424, + 0, + 0.28443410992622375, + 0, + 0, + 0.0014393558958545327, + 0, + 0, + -0.009063852950930595, + 0, + 0, + 0, + 0, + 0.001169737195596099, + 0, + 0, + 0, + 0, + 0, + 0, + 0.06898342072963715, + 0, + 0.007991905324161053, + 0, + 0.006260615773499012, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.037955716252326965, + 0.3505173921585083, + 0, + 0, + 0, + -0.338177889585495, + 0, + 0.158599853515625, + 0, + 0, + 0.01131439208984375, + 0, + 0.006751265376806259, + 0, + 0, + 0, + -0.04573351889848709, + 0, + 0, + 0.04386100172996521, + 0.11277603358030319, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + -0.0003205372195225209, + 0, + 0, + 0, + -0.005409737583249807, + -0.009204162284731865, + 0, + 0, + 0, + 0.004804544150829315, + -0.005810749251395464, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.09645535796880722, + 0, + 0, + 0, + 0.032931435853242874, + 0, + 0, + 0, + 0, + 0.028524864464998245, + 0.09402520954608917, + 0, + 0.008998546749353409, + 0, + -0.03251685947179794, + 0, + 0, + 0, + 0, + 0, + 0, + 0.037343256175518036 + ], + [ + 0.0045840502716600895, + 0, + 0, + 0.009021798148751259, + 0, + -0.004217533860355616, + 0, + 0.0025705555453896523, + 0, + -0.035309672355651855, + -0.09942735731601715, + -0.004700342193245888, + 0, + 0, + 0, + 0.018288278952240944, + 0.0004021169152110815, + 0, + 0, + 0, + 0, + -0.005593586713075638, + 0.0018821493722498417, + 0, + 0, + 0, + 0, + -0.04561242088675499, + -0.01815006509423256, + 0.016583485528826714, + -0.020843051373958588, + 0, + 0, + 0, + -0.006372869946062565, + 0.04272369295358658, + 0, + 0.0013309348141774535, + 0, + 0, + 0, + -0.0031638317741453648, + 0, + 0.08714215457439423, + 0, + -0.005442100111395121, + -0.00039313771412707865, + 0.0014464370906352997, + 0, + 0, + 0, + 0, + -0.03132649511098862, + 0, + 0, + -0.007972904480993748, + 0.01753396727144718, + 0, + -0.0011563192820176482, + 0, + 0.0017362519865855575, + 0, + -0.0004587600124068558, + 0, + 0.0038881096988916397, + -0.008516360074281693, + 0, + -0.008183307014405727, + -0.010095844976603985, + 0, + 0.10722006857395172, + -0.002898464212194085, + 0, + 0.0012827662285417318, + 0.00004252225699019618, + 0, + 0.07567721605300903, + -0.030121177434921265, + 0, + -0.0010666534071788192, + 0.0006539365276694298, + 0, + 0, + -0.0011567147448658943, + -0.021622339263558388, + 0.028687214478850365, + 0, + 0, + 0.018764594569802284, + 0.010613140650093555, + 0.019510075449943542, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + -0.005288010463118553, + 0, + 0, + -0.016743114218115807, + -0.015873711556196213, + -0.009877816773951054, + 0.0003150522243231535, + 0.023689158260822296, + 0, + -0.00033418016391806304, + 0, + 0.04904749244451523, + 0, + 0, + 0, + 0.0006500506424345076, + 0.000622213410679251, + 0.00738720316439867, + 0, + 0.0012243357487022877, + 0.009066173806786537, + -0.12073952704668045, + 0.0010678119724616408, + 0.006296947598457336, + 0.002682592486962676, + -0.00444818427786231, + -0.0023324599023908377, + 0, + 0, + -0.5609893798828125, + -0.008780602365732193, + 0.015986066311597824, + 0, + -0.02213476411998272, + 0, + 0, + 0.0006705078994855285, + -0.0011221399763599038, + 0, + 0, + 0.025299811735749245, + 0, + -0.008218510076403618 + ], + [ + -0.0034782839938998222, + 0.0022423912305384874, + 0, + 0, + 0, + 0, + 0, + -0.05859537422657013, + 0.0421387143433094, + 0.26256099343299866, + 0, + 0, + 0, + 0, + 0, + -0.10330676287412643, + 0.012355834245681763, + 0.0013472040882334113, + 0, + 0.019914263859391212, + 0.005261276848614216, + 0.001149827498011291, + -0.03320133313536644, + 0, + 0, + 0, + 0, + 0.32198745012283325, + 0, + 0.05401667580008507, + 0.04610951617360115, + -0.2326284795999527, + 0, + 0.0000856258993735537, + 0, + 0.074106365442276, + 0, + 0.044469863176345825, + 0, + 0, + 0, + -0.006453251000493765, + 0, + -0.018431225791573524, + 0, + 0, + 0, + 0, + -0.005704954732209444, + 0, + 0, + 0, + 0, + 0, + 0, + 0.13457728922367096, + 0, + -0.029186677187681198, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + -0.022995056584477425, + 0, + 0, + 0, + -0.09004921466112137, + 0.24257110059261322, + 0.02852930873632431, + 0, + 0.021270141005516052, + -0.13564155995845795, + -0.03098711557686329, + 0, + 0, + 0, + 0.0486220121383667, + -0.001395023544318974, + 0, + 0.04929636791348457, + 0, + 0, + -0.13068373501300812, + 0, + 0, + 0.016955919563770294, + 0.03848254308104515, + 0, + -0.011160435155034065, + 0, + 0, + -0.0002991429646499455, + 0, + 0, + 0, + 0.01138608530163765, + -0.020150866359472275, + 0, + 0, + -0.007353566121309996, + -0.021389631554484367, + -0.042083244770765305, + 0.13586723804473877, + 0.005315479822456837, + 0, + 0.008157049305737019, + 0.022239860147237778, + 0, + 0, + 0.01896926946938038, + 0.0018052944215014577, + 0.0016496418975293636, + 0.0005593635141849518, + 0, + 0, + 0.07655386626720428, + 0, + 0, + 0, + 0.02781328558921814, + 0.04012482985854149, + 0, + 0, + 0, + 0.10631410032510757, + 0.03608629107475281, + 0, + -0.02651066705584526, + 0, + -0.0690990686416626, + 0, + 0, + 0, + 0, + 0, + 0, + 0.1022648885846138 + ] + ] + } + ], + "layout": { + "coloraxis": { + "cmid": 0, + "colorscale": [ + [ + 0, + "rgb(103,0,31)" + ], + [ + 0.1, + "rgb(178,24,43)" + ], + [ + 0.2, + "rgb(214,96,77)" + ], + [ + 0.3, + "rgb(244,165,130)" + ], + [ + 0.4, + "rgb(253,219,199)" + ], + [ + 0.5, + "rgb(247,247,247)" + ], + [ + 0.6, + "rgb(209,229,240)" + ], + [ + 0.7, + "rgb(146,197,222)" + ], + [ + 0.8, + "rgb(67,147,195)" + ], + [ + 0.9, + "rgb(33,102,172)" + ], + [ + 1, + "rgb(5,48,97)" + ] + ] + }, + "template": { + "data": { + "bar": [ + { + "error_x": { + "color": "#2a3f5f" + }, + "error_y": { + "color": "#2a3f5f" + }, + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "bar" + } + ], + "barpolar": [ + { + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "barpolar" + } + ], + "carpet": [ + { + "aaxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "baxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "type": "carpet" + } + ], + "choropleth": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "choropleth" + } + ], + "contour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "contour" + } + ], + "contourcarpet": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "contourcarpet" + } + ], + "heatmap": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmap" + } + ], + "heatmapgl": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmapgl" + } + ], + "histogram": [ + { + "marker": { + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "histogram" + } + ], + "histogram2d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2d" + } + ], + "histogram2dcontour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2dcontour" + } + ], + "mesh3d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "mesh3d" + } + ], + "parcoords": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "parcoords" + } + ], + "pie": [ + { + "automargin": true, + "type": "pie" + } + ], + "scatter": [ + { + "fillpattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + }, + "type": "scatter" + } + ], + "scatter3d": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatter3d" + } + ], + "scattercarpet": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattercarpet" + } + ], + "scattergeo": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergeo" + } + ], + "scattergl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergl" + } + ], + "scattermapbox": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermapbox" + } + ], + "scatterpolar": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolar" + } + ], + "scatterpolargl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolargl" + } + ], + "scatterternary": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterternary" + } + ], + "surface": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "surface" + } + ], + "table": [ + { + "cells": { + "fill": { + "color": "#EBF0F8" + }, + "line": { + "color": "white" + } + }, + "header": { + "fill": { + "color": "#C8D4E3" + }, + "line": { + "color": "white" + } + }, + "type": "table" + } + ] + }, + "layout": { + "annotationdefaults": { + "arrowcolor": "#2a3f5f", + "arrowhead": 0, + "arrowwidth": 1 + }, + "autotypenumbers": "strict", + "coloraxis": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "colorscale": { + "diverging": [ + [ + 0, + "#8e0152" + ], + [ + 0.1, + "#c51b7d" + ], + [ + 0.2, + "#de77ae" + ], + [ + 0.3, + "#f1b6da" + ], + [ + 0.4, + "#fde0ef" + ], + [ + 0.5, + "#f7f7f7" + ], + [ + 0.6, + "#e6f5d0" + ], + [ + 0.7, + "#b8e186" + ], + [ + 0.8, + "#7fbc41" + ], + [ + 0.9, + "#4d9221" + ], + [ + 1, + "#276419" + ] + ], + "sequential": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "sequentialminus": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ] + }, + "colorway": [ + "#636efa", + "#EF553B", + "#00cc96", + "#ab63fa", + "#FFA15A", + "#19d3f3", + "#FF6692", + "#B6E880", + "#FF97FF", + "#FECB52" + ], + "font": { + "color": "#2a3f5f" + }, + "geo": { + "bgcolor": "white", + "lakecolor": "white", + "landcolor": "#E5ECF6", + "showlakes": true, + "showland": true, + "subunitcolor": "white" + }, + "hoverlabel": { + "align": "left" + }, + "hovermode": "closest", + "mapbox": { + "style": "light" + }, + "paper_bgcolor": "white", + "plot_bgcolor": "#E5ECF6", + "polar": { + "angularaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "radialaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "scene": { + "xaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "yaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "zaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + } + }, + "shapedefaults": { + "line": { + "color": "#2a3f5f" + } + }, + "ternary": { + "aaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "baxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "caxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "title": { + "x": 0.05 + }, + "xaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + }, + "yaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + } + } + }, + "title": { + "text": "attribution patching" + }, + "xaxis": { + "anchor": "y", + "constrain": "domain", + "domain": [ + 0, + 1 + ], + "scaleanchor": "y", + "title": { + "text": "Feature Idx" + } + }, + "yaxis": { + "anchor": "x", + "autorange": "reversed", + "constrain": "domain", + "domain": [ + 0, + 1 + ], + "title": { + "text": "Prompt Idx" + } + } + } + }, + "text/html": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def attr_patch_sae_acts(\n", + " clean_cache: ActivationCache, \n", + " clean_grad_cache: ActivationCache,\n", + " site: str, layer: int\n", + " ):\n", + " clean_sae_acts_post = clean_cache[utils.get_act_name(site, layer) + \".hook_sae_acts_post\"] \n", + " clean_grad_sae_acts_post = clean_grad_cache[utils.get_act_name(site, layer) + \".hook_sae_acts_post\"] \n", + " sae_act_attr = clean_grad_sae_acts_post * (0 - clean_sae_acts_post)\n", + " return sae_act_attr\n", + "\n", + "site = \"z\"\n", + "layer = 5\n", + "sae_act_attr = attr_patch_sae_acts(clean_cache, clean_grad_cache, site, layer)\n", + "\n", + "imshow(\n", + " sae_act_attr[:, s2_pos, all_live_features],\n", + " title=\"attribution patching\",\n", + " xaxis=\"Feature Idx\", yaxis=\"Prompt Idx\", x=list(map(str, all_live_features.tolist())))" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.plotly.v1+json": { + "config": { + "plotlyServerURL": "https://plot.ly" + }, + "data": [ + { + "hovertemplate": "Activation Patch=%{x}
Attribution Patch=%{y}", + "legendgroup": "", + "marker": { + "color": "#636efa", + "symbol": "circle" + }, + "mode": "markers", + "name": "", + "showlegend": false, + "type": "scattergl", + "x": [ + 0.0012617111206054688, + -9.5367431640625e-7, + -9.5367431640625e-7, + 0.0016908645629882812, + -0.0002231597900390625, + -9.5367431640625e-7, + -9.5367431640625e-7, + -0.00029659271240234375, + -9.5367431640625e-7, + -0.03279590606689453, + -0.07254886627197266, + -9.5367431640625e-7, + 0.00013065338134765625, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -0.014922142028808594, + -0.0044403076171875, + 0.0007047653198242188, + -0.00428009033203125, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -0.039069175720214844, + -9.5367431640625e-7, + -9.5367431640625e-7, + -0.007334709167480469, + 0.00033092498779296875, + -0.0017004013061523438, + 0.0026845932006835938, + 0.00043010711669921875, + -0.11128997802734375, + -0.0038976669311523438, + -0.006033897399902344, + -9.5367431640625e-7, + -0.00027751922607421875, + -9.5367431640625e-7, + 0.0006570816040039062, + -0.0004291534423828125, + -9.5367431640625e-7, + -0.0035734176635742188, + -0.0033063888549804688, + -9.5367431640625e-7, + -9.5367431640625e-7, + 0.0033960342407226562, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -0.0030546188354492188, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -0.0000972747802734375, + -0.0001811981201171875, + -9.5367431640625e-7, + -0.004569053649902344, + -0.013583183288574219, + -9.5367431640625e-7, + 0.02047252655029297, + -0.02572154998779297, + -9.5367431640625e-7, + -0.0006608963012695312, + -9.5367431640625e-7, + -9.5367431640625e-7, + 0.02255725860595703, + -0.05519580841064453, + -0.0033473968505859375, + -0.0000057220458984375, + -0.0026073455810546875, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -0.02097320556640625, + 0.008440971374511719, + -9.5367431640625e-7, + -0.004597663879394531, + 0.00159454345703125, + 0.0001544952392578125, + 0.005199432373046875, + 0.0007762908935546875, + -9.5367431640625e-7, + -9.5367431640625e-7, + -0.0032625198364257812, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -0.015192985534667969, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -0.018138885498046875, + -9.5367431640625e-7, + 0.010298728942871094, + -9.5367431640625e-7, + -0.0031423568725585938, + -9.5367431640625e-7, + 0.004242897033691406, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -0.010041236877441406, + 0.0010347366333007812, + 0.006011962890625, + -9.5367431640625e-7, + -9.5367431640625e-7, + 0.00301361083984375, + -0.04584026336669922, + 0.0002079010009765625, + -9.5367431640625e-7, + -9.5367431640625e-7, + -0.0002574920654296875, + -9.5367431640625e-7, + -9.5367431640625e-7, + -9.5367431640625e-7, + -0.45942211151123047, + -0.0008325576782226562, + 0.00041484832763671875, + -9.5367431640625e-7, + -0.023777008056640625, + 0.0000514984130859375, + -9.5367431640625e-7, + -9.5367431640625e-7, + -0.00030422210693359375, + 0.0006666183471679688, + -9.5367431640625e-7, + 0.004633903503417969, + -9.5367431640625e-7, + -0.008234977722167969, + 0.000003814697265625, + 0.000003814697265625, + -0.00208282470703125, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + -0.0012912750244140625, + -0.01760101318359375, + 0.000003814697265625, + 0.057277679443359375, + 0.013429641723632812, + 0.000003814697265625, + 0.000003814697265625, + -0.0000457763671875, + -0.0027828216552734375, + -0.0055084228515625, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + -0.2744255065917969, + 0.000003814697265625, + 0.000003814697265625, + 0.0021514892578125, + 0.000003814697265625, + 0.06994247436523438, + 0.0048542022705078125, + 0.000003814697265625, + 0.000003814697265625, + -0.0567169189453125, + 0.000003814697265625, + 0.012315750122070312, + 0.0066585540771484375, + 0.07937240600585938, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.028867721557617188, + 0.000003814697265625, + 0.000003814697265625, + 0.0074901580810546875, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.009624481201171875, + 0.000003814697265625, + -0.009510040283203125, + 0.0032100677490234375, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.10918617248535156, + 0.000003814697265625, + 0.026102066040039062, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.000946044921875, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + -0.041675567626953125, + 0.000003814697265625, + 0.000003814697265625, + -0.0066776275634765625, + 0.03926849365234375, + 0.03615379333496094, + 0.027612686157226562, + 0.000003814697265625, + -0.0004673004150390625, + -0.1435985565185547, + -0.00030517578125, + 0.059326171875, + 0.000003814697265625, + 0.000003814697265625, + 0.020435333251953125, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + -0.11923980712890625, + -0.009393692016601562, + 0.000003814697265625, + 0.011783599853515625, + 0.06122589111328125, + 0.000003814697265625, + 0.000003814697265625, + 0.0002918243408203125, + 0.000003814697265625, + 0.000003814697265625, + 0.001491546630859375, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + -0.0050716400146484375, + 0.000003814697265625, + 0.000003814697265625, + 0.025064468383789062, + 0.000003814697265625, + -0.0467529296875, + 0.000003814697265625, + 0.000003814697265625, + 0.0014934539794921875, + 0.00043487548828125, + 0.028188705444335938, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.001995086669921875, + 0.000003814697265625, + 0.000003814697265625, + 0.13014602661132812, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.0005893707275390625, + 0.012182235717773438, + 0.000003814697265625, + 0.000003814697265625, + 0.11103057861328125, + 0.042850494384765625, + 0.030099868774414062, + 0.000003814697265625, + -0.0047321319580078125, + 0.0000133514404296875, + -0.0320587158203125, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.000003814697265625, + 0.031030654907226562, + 0.007018089294433594, + 0, + 0, + 0.0028057098388671875, + 0, + 0, + 0, + -0.010999679565429688, + 0, + -0.1419973373413086, + -0.24188613891601562, + 0, + 0.0003147125244140625, + 0, + 0, + 0, + 0.009432792663574219, + 0, + -0.000125885009765625, + 0, + 0, + 0.00017070770263671875, + 0.011651992797851562, + -0.00225830078125, + 0, + -0.0014581680297851562, + 0.00020122528076171875, + -0.030771255493164062, + -0.03744316101074219, + -0.034499168395996094, + -0.00374603271484375, + 0, + 0.0011348724365234375, + 0, + -0.0302276611328125, + -0.08229637145996094, + -0.00048160552978515625, + 0, + -0.00640869140625, + 0.0001277923583984375, + 0, + -0.0008974075317382812, + 0.00022983551025390625, + -0.2322559356689453, + -0.0050449371337890625, + -0.010677337646484375, + 0, + 0.014942169189453125, + 0, + 0.0008764266967773438, + 0.00417327880859375, + 0, + -0.015301704406738281, + 0, + -0.0008974075317382812, + -0.04426097869873047, + 0.005242347717285156, + 0, + 0, + 0, + -0.009447097778320312, + 0, + 0, + -0.0011806488037109375, + 0, + -0.0045909881591796875, + 0.015285491943359375, + -0.034976959228515625, + -0.013401985168457031, + 0, + 0.1357421875, + -0.09111690521240234, + 0, + 0.00013065338134765625, + 0.0002460479736328125, + 0, + 0.04656982421875, + -0.09346866607666016, + 0, + -0.005030632019042969, + 0.0001125335693359375, + 0, + 0, + 0, + -0.07491683959960938, + 0.006598472595214844, + 0, + -0.014060020446777344, + -0.008306503295898438, + -0.0054874420166015625, + -0.0004930496215820312, + 0, + 0, + 0, + -0.008953094482421875, + 0, + 0, + 0, + -0.03713417053222656, + 0, + 0, + 0, + 0, + -0.028200149536132812, + 0, + 0.036255836486816406, + 0, + -0.03178215026855469, + 0, + -0.012192726135253906, + -0.002147674560546875, + -0.0005474090576171875, + -0.0021409988403320312, + -0.030725479125976562, + 0, + 0.0008029937744140625, + 0, + 0, + 0, + -0.29135894775390625, + 0, + 0, + 0, + 0, + 0, + -0.0027914047241210938, + -0.00022125244140625, + -0.8653240203857422, + 0, + -0.05593109130859375, + 0, + -0.04123210906982422, + 0, + 0, + 0.015351295471191406, + 0, + 0, + 0, + 0.018423080444335938, + -0.0000476837158203125, + -0.0023584365844726562, + 9.5367431640625e-7, + 9.5367431640625e-7, + 0.0001983642578125, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + -0.010341644287109375, + 0.07198715209960938, + 0.14725303649902344, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 0.0002918243408203125, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 0.011704444885253906, + 9.5367431640625e-7, + 9.5367431640625e-7, + -0.3150959014892578, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + -0.039947509765625, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 0.15607547760009766, + 9.5367431640625e-7, + 0.09917640686035156, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 0.019521713256835938, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 0.012205123901367188, + 9.5367431640625e-7, + -0.0005893707275390625, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 0.07062149047851562, + 0.000492095947265625, + 0.014776229858398438, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 0.0557098388671875, + 0.15409469604492188, + 9.5367431640625e-7, + 9.5367431640625e-7, + -0.0007076263427734375, + -0.24256324768066406, + 9.5367431640625e-7, + 0.0858917236328125, + 9.5367431640625e-7, + 9.5367431640625e-7, + 0.007343292236328125, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + -0.11646080017089844, + 9.5367431640625e-7, + 9.5367431640625e-7, + 0.05528736114501953, + 0.0847921371459961, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 0.00428009033203125, + -0.0056171417236328125, + 9.5367431640625e-7, + 9.5367431640625e-7, + 0.0066967010498046875, + 9.5367431640625e-7, + -0.006005287170410156, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 0.01735687255859375, + -0.0037336349487304688, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 0.09533309936523438, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 0.009324073791503906, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 0.007989883422851562, + 9.5367431640625e-7, + 0.0064525604248046875, + 9.5367431640625e-7, + -0.06574440002441406, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + 9.5367431640625e-7, + -0.0009012222290039062, + 0.00001239776611328125, + 0.00001239776611328125, + -0.0006313323974609375, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + -0.000461578369140625, + 0.00001239776611328125, + -0.055993080139160156, + -0.24974536895751953, + 0.00001239776611328125, + 0.0011262893676757812, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + -0.0025796890258789062, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + -0.030013084411621094, + -0.012925148010253906, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + -0.0253448486328125, + 0.00001239776611328125, + 0.0012464523315429688, + 0.021536827087402344, + 0.00001239776611328125, + 0.00001239776611328125, + -0.00009822845458984375, + 0.00001239776611328125, + -0.09924793243408203, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.006188392639160156, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + -0.0010576248168945312, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.0008172988891601562, + 0.00001239776611328125, + 0.00001239776611328125, + -0.0020704269409179688, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + -0.09985160827636719, + 0.00001239776611328125, + 0.00001239776611328125, + 0.036945343017578125, + 0.025011062622070312, + 0.00001239776611328125, + 0.004599571228027344, + 0.00001239776611328125, + 0.00001239776611328125, + 0.027939796447753906, + -0.07974910736083984, + 0.00001239776611328125, + 0.00001239776611328125, + -0.00038242340087890625, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + -0.035175323486328125, + 0.00001239776611328125, + 0.00001239776611328125, + -0.0047245025634765625, + -0.008166313171386719, + -0.008578300476074219, + 0.0018529891967773438, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + -0.0016679763793945312, + 0.00001239776611328125, + 0.00001239776611328125, + -0.0028676986694335938, + -0.04880046844482422, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + -0.0053462982177734375, + 0.00001239776611328125, + 0.1658468246459961, + 0.00001239776611328125, + -0.0024824142456054688, + 0.00001239776611328125, + 0.025139808654785156, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + -0.027915000915527344, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + -0.14544200897216797, + 0.00001239776611328125, + 0.020270347595214844, + 0.007473945617675781, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + -0.8424196243286133, + 0.00001239776611328125, + -0.007409095764160156, + -0.00318145751953125, + -0.015982627868652344, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00001239776611328125, + 0.00034046173095703125, + 0.10727787017822266, + 0.00001239776611328125, + 0.00001239776611328125, + -0.00000762939453125, + -0.00000762939453125, + 0.0019397735595703125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.022940635681152344, + 0.07428932189941406, + 0.29994869232177734, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.016974449157714844, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.4772310256958008, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + 0.05463600158691406, + -0.00000762939453125, + -0.00000762939453125, + 0.004734992980957031, + -0.12352275848388672, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + 0.15236186981201172, + -0.00000762939453125, + 0.27855396270751953, + -0.00000762939453125, + -0.00000762939453125, + 0.001430511474609375, + -0.00000762939453125, + -0.00000762939453125, + -0.016387939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + 0.0008668899536132812, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + 0.06814861297607422, + -0.00000762939453125, + 0.00351715087890625, + -0.00000762939453125, + 0.0061588287353515625, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + 0.02480602264404297, + 0.31668567657470703, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.4413900375366211, + -0.00000762939453125, + 0.1517791748046875, + -0.00000762939453125, + -0.00000762939453125, + 0.010898590087890625, + -0.00000762939453125, + 0.006583213806152344, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.04946422576904297, + -0.00000762939453125, + -0.00000762939453125, + 0.040429115295410156, + 0.1020956039428711, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.0008649826049804688, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.0054836273193359375, + -0.010519981384277344, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + 0.004588127136230469, + -0.006558418273925781, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + 0.07750797271728516, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + 0.03235149383544922, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + 0.02773571014404297, + 0.08978557586669922, + -0.00000762939453125, + 0.008780479431152344, + -0.00000762939453125, + -0.0327301025390625, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + -0.00000762939453125, + 0.035370826721191406, + 0.0041904449462890625, + -0.0000057220458984375, + -0.0000057220458984375, + 0.008458137512207031, + -0.0000057220458984375, + -0.0042858123779296875, + -0.0000057220458984375, + 0.002468109130859375, + -0.0000057220458984375, + -0.03716564178466797, + -0.10456657409667969, + -0.0047702789306640625, + -0.0000057220458984375, + -0.0000057220458984375, + -0.0000057220458984375, + 0.017671585083007812, + 0.0004062652587890625, + -0.0000057220458984375, + -0.0000057220458984375, + -0.0000057220458984375, + -0.0000057220458984375, + -0.00655364990234375, + 0.001873016357421875, + -0.0000057220458984375, + -0.0000057220458984375, + -0.0000057220458984375, + -0.0000057220458984375, + -0.04653644561767578, + -0.01836395263671875, + 0.014448165893554688, + -0.0209197998046875, + -0.0000057220458984375, + -0.0000057220458984375, + -0.0000057220458984375, + -0.006618499755859375, + 0.02408599853515625, + -0.0000057220458984375, + 0.0012884140014648438, + -0.0000057220458984375, + -0.0000057220458984375, + -0.0000057220458984375, + -0.0050144195556640625, + -0.0000057220458984375, + 0.036708831787109375, + -0.0000057220458984375, + -0.0056591033935546875, + -0.0004215240478515625, + 0.0014057159423828125, + -0.0000057220458984375, + -0.0000057220458984375, + -0.0000057220458984375, + -0.0000057220458984375, + -0.033232688903808594, + -0.0000057220458984375, + -0.0000057220458984375, + -0.008130073547363281, + 0.016930580139160156, + -0.0000057220458984375, + -0.0012025833129882812, + -0.0000057220458984375, + 0.000545501708984375, + -0.0000057220458984375, + -0.0004673004150390625, + -0.0000057220458984375, + 0.0038089752197265625, + -0.008646011352539062, + -0.0000057220458984375, + -0.008909225463867188, + -0.011255264282226562, + -0.0000057220458984375, + 0.0925750732421875, + -0.0064563751220703125, + -0.0000057220458984375, + 0.0011615753173828125, + 0.00002956390380859375, + -0.0000057220458984375, + 0.07063961029052734, + -0.030902862548828125, + -0.0000057220458984375, + -0.0010814666748046875, + 0.00038909912109375, + -0.0000057220458984375, + -0.0000057220458984375, + -0.0013380050659179688, + -0.022397994995117188, + 0.027740478515625, + -0.0000057220458984375, + -0.0000057220458984375, + 0.01797771453857422, + 0.009552955627441406, + 0.01857471466064453, + -0.0000057220458984375, + -0.0000057220458984375, + -0.0000057220458984375, + -0.0000057220458984375, + -0.0000057220458984375, + -0.0000057220458984375, + -0.0000057220458984375, + -0.0055103302001953125, + -0.0000057220458984375, + -0.0000057220458984375, + -0.01697063446044922, + -0.0159149169921875, + -0.011240959167480469, + 0.000301361083984375, + 0.020501136779785156, + -0.0000057220458984375, + -0.0006427764892578125, + -0.0000057220458984375, + 0.04800224304199219, + -0.0000057220458984375, + -0.0000057220458984375, + -0.0000057220458984375, + -0.00001811981201171875, + 0.0005950927734375, + 0.00732421875, + -0.0000057220458984375, + 0.001216888427734375, + 0.00897216796875, + -0.1255035400390625, + 0.001003265380859375, + 0.006274223327636719, + 0.0026502609252929688, + -0.00449371337890625, + -0.0023517608642578125, + -0.0000057220458984375, + -0.0000057220458984375, + -0.6521244049072266, + -0.009072303771972656, + 0.013387680053710938, + -0.0000057220458984375, + -0.022745132446289062, + -0.0000057220458984375, + -0.0000057220458984375, + 0.000606536865234375, + -0.0011501312255859375, + -0.0000057220458984375, + -0.0000057220458984375, + 0.023046493530273438, + -0.0000057220458984375, + -0.008263587951660156, + -0.0037221908569335938, + 0.00225830078125, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.05941295623779297, + 0.04140281677246094, + 0.24284648895263672, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.11462688446044922, + 0.012240409851074219, + 0.0012884140014648438, + -0.00001049041748046875, + 0.01781749725341797, + 0.005211830139160156, + -0.0016298294067382812, + -0.2994966506958008, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + 0.26962947845458984, + -0.00001049041748046875, + 0.050202369689941406, + 0.04053211212158203, + -0.30355358123779297, + -0.00001049041748046875, + -0.0013666152954101562, + -0.00001049041748046875, + 0.06442928314208984, + -0.00001049041748046875, + 0.04406547546386719, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00763702392578125, + -0.00001049041748046875, + -0.03402233123779297, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.005751609802246094, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + 0.13191986083984375, + -0.00001049041748046875, + -0.031653404235839844, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.02394390106201172, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.1073293685913086, + 0.20270729064941406, + 0.02746295928955078, + -0.00001049041748046875, + 0.020377159118652344, + -0.31055259704589844, + -0.043480873107910156, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + 0.04507160186767578, + -0.0014734268188476562, + -0.00001049041748046875, + 0.048813819885253906, + -0.00001049041748046875, + -0.00001049041748046875, + -0.1407604217529297, + -0.00001049041748046875, + -0.00001049041748046875, + 0.013548851013183594, + 0.016210556030273438, + -0.00001049041748046875, + -0.011261940002441406, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00029277801513671875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + 0.008993148803710938, + -0.020813941955566406, + -0.00001049041748046875, + -0.00001049041748046875, + -0.008435249328613281, + -0.021961212158203125, + -0.04410362243652344, + 0.1307668685913086, + 0.005297660827636719, + -0.00001049041748046875, + 0.006031990051269531, + 0.016150474548339844, + -0.00001049041748046875, + -0.00001049041748046875, + 0.01802349090576172, + 0.0018205642700195312, + 0.0016574859619140625, + 0.0005712509155273438, + -0.00001049041748046875, + -0.00001049041748046875, + 0.02598094940185547, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + 0.02737903594970703, + 0.039580345153808594, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + 0.09876728057861328, + 0.035803794860839844, + -0.00001049041748046875, + -0.027251243591308594, + -0.00001049041748046875, + -0.07061004638671875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + -0.00001049041748046875, + 0.08719158172607422 + ], + "xaxis": "x", + "y": [ + 0.001567811705172062, + 0, + 0, + 0.001697835512459278, + 0.00011560246639419347, + 0, + 0, + -0.0002851475146599114, + 0, + -0.030827227979898453, + -0.06409652531147003, + 0, + 0.00015167289529927075, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + -0.013627146370708942, + -0.004393726587295532, + 0.0015328703448176384, + -0.0038613511715084314, + 0, + 0, + 0, + 0, + -0.02049136720597744, + 0, + 0, + -0.007114107254892588, + 0.0003477374848444015, + -0.001384311355650425, + 0.003183899214491248, + 0.0004558839718811214, + -0.059277813881635666, + -0.0035793157294392586, + -0.00589390005916357, + 0, + -0.0001910730206873268, + 0, + 0.0006608504336327314, + -0.0004212319909129292, + 0, + -0.003545185085386038, + -0.00327106611803174, + 0, + 0, + 0.0040074847638607025, + 0, + 0, + 0, + -0.0026069351006299257, + 0, + 0, + 0, + -0.00008433026232523844, + -0.00018646706303115934, + 0, + -0.00439279293641448, + -0.013254894874989986, + 0, + 0.050094299018383026, + -0.021308520808815956, + 0, + -0.0006410681526176631, + 0, + 0, + 0.02329532988369465, + -0.05166983604431152, + -0.002982117934152484, + -0.000014124364497547504, + -0.0020334068685770035, + 0, + 0, + 0, + -0.02020590752363205, + 0.00998645182698965, + 0, + -0.004585121292620897, + 0.005916096270084381, + 0.0018219061894342303, + 0.005700498353689909, + 0.0008085825829766691, + 0, + 0, + -0.0032405084930360317, + 0, + 0, + 0, + -0.014961971901357174, + 0, + 0, + 0, + 0, + -0.016915086656808853, + 0, + 0.016825370490550995, + 0, + -0.00311169121414423, + 0, + 0.005266942549496889, + 0, + 0, + 0, + -0.009660078212618828, + 0.0010975055629387498, + 0.006078756880015135, + 0, + 0, + 0.003166533075273037, + -0.044512320309877396, + 0.0002630578528624028, + 0, + 0, + -0.00025422731414437294, + 0, + 0, + 0, + -0.3718416392803192, + -0.0008081833366304636, + 0.00043700754758901894, + 0, + -0.023154418915510178, + 0.00004691413778346032, + 0, + 0, + -0.0002914638607762754, + 0.0006733346963301301, + 0, + 0.008972969837486744, + 0, + -0.008168808184564114, + 0, + 0, + -0.0006953283445909619, + 0, + 0, + 0, + -0.001286927843466401, + -0.017273705452680588, + 0, + 0.05898163467645645, + 0.013462062925100327, + 0, + 0, + -0.00003325308352941647, + -0.0027551515959203243, + -0.004652985371649265, + 0, + 0, + 0, + 0, + 0, + 0, + -0.21421866118907928, + 0, + 0, + 0.002191215055063367, + 0, + 0.07645706832408905, + 0.0052618952468037605, + 0, + 0, + -0.020269982516765594, + 0, + 0.013446477241814137, + 0.0068704248405992985, + 0.08710267394781113, + 0, + 0, + 0, + 0, + 0.028982989490032196, + 0, + 0, + 0.014961526729166508, + 0, + 0, + 0, + 0, + 0.011233230121433735, + 0, + -0.009112805128097534, + 0.003226917004212737, + 0, + 0, + 0, + 0.112985759973526, + 0, + 0.028253009542822838, + 0, + 0, + 0, + 0.0009787877788767219, + 0, + 0, + 0, + 0, + -0.03986968472599983, + 0, + 0, + -0.006135094445198774, + 0.04977395758032799, + 0.0397123359143734, + 0.027974072843790054, + 0, + -0.00044811973930336535, + -0.10083132237195969, + 0.000008234118467953522, + 0.06165996566414833, + 0, + 0, + 0.021058127284049988, + 0, + 0, + 0, + 0, + 0, + -0.08074336498975754, + -0.009298793971538544, + 0, + 0.012482613325119019, + 0.06513619422912598, + 0, + 0, + 0.00029019018984399736, + 0, + 0, + 0.0014882637187838554, + 0, + 0, + 0, + -0.004803473129868507, + 0, + 0, + 0.025678949430584908, + 0, + -0.04240157827734947, + 0, + 0, + 0.0015190609265118837, + 0.0006482255994342268, + 0.03654245659708977, + 0, + 0, + 0, + 0, + 0, + 0.0020186977926641703, + 0, + 0, + 0.17831696569919586, + 0, + 0, + 0, + 0.0005887048901058733, + 0.012331255711615086, + 0, + 0, + 0.11619613319635391, + 0.04687207192182541, + 0.03033648431301117, + 0, + -0.004195880610495806, + 0.00006391256465576589, + -0.03162289038300514, + 0, + 0, + 0, + 0, + 0, + 0, + 0.03672636300325394, + 0.00788492988795042, + 0, + 0, + 0.003685369621962309, + 0, + 0, + 0, + -0.010384900495409966, + 0, + -0.1327948272228241, + -0.22788244485855103, + 0, + 0.0003893508983310312, + 0, + 0, + 0, + 0.009530982933938503, + 0, + -0.0001286355109186843, + 0, + 0, + 0.0001596187794348225, + 0.011789986863732338, + -0.0022452236153185368, + 0, + -0.0014552043285220861, + 0.0002036036894423887, + -0.03003234602510929, + -0.036742936819791794, + -0.028862446546554565, + -0.003727517556399107, + 0, + 0.0011460097739472985, + 0, + -0.027142589911818504, + -0.054151974618434906, + -0.0004727205669041723, + 0, + -0.006094601005315781, + 0.00013960858632344753, + 0, + -0.0003665595140773803, + 0.00028091753483749926, + -0.17846877872943878, + -0.004990901332348585, + -0.010615025646984577, + 0, + 0.015916047617793083, + 0, + 0.0008773574372753501, + 0.004459311719983816, + 0, + -0.015235064551234245, + 0, + -0.0008741968194954097, + -0.04074608162045479, + 0.007227533031255007, + 0, + 0, + 0, + -0.007763775996863842, + 0, + 0, + -0.0011336231837049127, + 0, + -0.004542750306427479, + 0.016146792098879814, + -0.032868705689907074, + -0.013282506726682186, + 0, + 0.1884474903345108, + -0.07819699496030807, + 0, + 0.00013099861098453403, + 0.00024322106037288904, + 0, + 0.04764547944068909, + -0.09056885540485382, + 0, + -0.005007788073271513, + 0.000487087934743613, + 0, + 0, + 0, + -0.07196655869483948, + 0.007451012264937162, + 0, + -0.013892672955989838, + -0.005596193019300699, + -0.005349555052816868, + -0.00015437132969964296, + 0, + 0, + 0, + -0.00894666463136673, + 0, + 0, + 0, + -0.036862581968307495, + 0, + 0, + 0, + 0, + -0.026162482798099518, + 0, + 0.046491872519254684, + 0, + -0.030160455033183098, + 0, + -0.009029642678797245, + -0.0021479984279721975, + -0.0005375721957534552, + -0.002135993679985404, + -0.027962258085608482, + 0, + 0.0008057129452936351, + 0, + 0, + 0, + -0.26795026659965515, + 0, + 0, + 0, + 0, + 0, + -0.0027670287527143955, + -0.0002252299600513652, + -0.7548060417175293, + 0, + -0.05009680241346359, + 0, + -0.03914204612374306, + 0, + 0, + 0.016279445961117744, + 0, + 0, + 0, + 0.025662390515208244, + -0.000049459828005637974, + -0.0023572721984237432, + 0, + 0, + 0.0009027881897054613, + 0, + 0, + 0, + 0, + -0.01007400918751955, + 0.07334298640489578, + 0.15174342691898346, + 0, + 0, + 0, + 0, + 0, + 0.0007311829249374568, + 0, + 0, + 0, + 0.011839455924928188, + 0, + 0, + -0.2282165139913559, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + -0.017542533576488495, + 0, + 0, + 0, + 0.1636323779821396, + 0, + 0.10289037227630615, + 0, + 0, + 0, + 0, + 0, + 0.024433566257357597, + 0, + 0, + 0, + 0, + 0.013018166646361351, + 0, + -0.0005916667287237942, + 0, + 0, + 0, + 0, + 0.07111621648073196, + 0.0004984873230569065, + 0.015917964279651642, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.06262800097465515, + 0.17253385484218597, + 0, + 0, + -0.0007970984443090856, + -0.1451263427734375, + 0, + 0.08718064427375793, + 0, + 0, + 0.007446629460901022, + 0, + 0, + 0, + 0, + 0, + -0.09546831995248795, + 0, + 0, + 0.06110787391662598, + 0.08931172639131546, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.005256101489067078, + -0.00553735950961709, + 0, + 0, + 0.006732907146215439, + 0, + -0.005547903478145599, + 0, + 0, + 0, + 0.01766844280064106, + -0.0034187675919383764, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.1122211441397667, + 0, + 0, + 0, + 0.009442206472158432, + 0, + 0, + 0, + 0, + 0, + 0.00800288561731577, + 0, + 0.006613056641072035, + 0, + -0.06462590396404266, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + -0.0009047402418218553, + 0, + 0, + -0.0005877931835129857, + 0, + 0, + 0, + -0.0004729636711999774, + 0, + -0.05036322772502899, + -0.24687804281711578, + 0, + 0.001115482416935265, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + -0.0024291854351758957, + 0, + 0, + 0, + -0.029154174029827118, + -0.011211197823286057, + 0, + 0, + 0, + 0, + 0, + -0.0075091151520609856, + 0, + 0.0037634933833032846, + 0.022711526602506638, + 0, + 0, + -0.00011145337339257821, + 0, + -0.08350298553705215, + 0, + 0, + 0, + 0, + 0, + 0.0063380529172718525, + 0, + 0, + 0, + 0, + 0, + 0, + -0.0010615212377160788, + 0, + 0, + 0, + 0.001314864493906498, + 0, + 0, + -0.0020079570822417736, + 0, + 0, + 0, + -0.095857173204422, + 0, + 0, + 0.04977884888648987, + 0.04924672096967697, + 0, + 0.00675918348133564, + 0, + 0, + 0.02823697216808796, + -0.07869893312454224, + 0, + 0, + -0.00039145027403719723, + 0, + 0, + 0, + -0.03502006456255913, + 0, + 0, + -0.004709419794380665, + -0.007543480955064297, + -0.007213911972939968, + 0.0026987697929143906, + 0, + 0, + 0, + -0.0016787010245025158, + 0, + 0, + -0.002866228111088276, + -0.04759479686617851, + 0, + 0, + 0, + 0, + -0.005348640959709883, + 0, + 0.17661413550376892, + 0, + -0.0024743194226175547, + 0, + 0.0269751138985157, + 0, + 0, + 0, + -0.025461290031671524, + 0, + 0, + 0, + 0, + 0, + -0.14607883989810944, + 0, + 0.020490022376179695, + 0.007573024369776249, + 0, + 0, + 0, + 0, + -0.8939738869667053, + 0, + -0.006900197826325893, + -0.0031849159859120846, + -0.015817783772945404, + 0, + 0, + 0, + 0, + 0, + 0.00032859406201168895, + 0.11629504710435867, + 0, + 0, + 0, + 0, + 0.0020032059401273727, + 0, + 0, + 0, + 0, + -0.02256190776824951, + 0.07616151124238968, + 0.3106333911418915, + 0, + 0, + 0, + 0, + 0, + -0.014044971205294132, + 0, + 0, + 0, + 0, + 0, + 0, + -0.3483165502548218, + 0, + 0, + 0, + 0, + 0.05930393189191818, + 0, + 0, + 0.004992437083274126, + -0.08404884487390518, + 0, + 0, + 0, + 0.16281214356422424, + 0, + 0.28443410992622375, + 0, + 0, + 0.0014393558958545327, + 0, + 0, + -0.009063852950930595, + 0, + 0, + 0, + 0, + 0.001169737195596099, + 0, + 0, + 0, + 0, + 0, + 0, + 0.06898342072963715, + 0, + 0.007991905324161053, + 0, + 0.006260615773499012, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.037955716252326965, + 0.3505173921585083, + 0, + 0, + 0, + -0.338177889585495, + 0, + 0.158599853515625, + 0, + 0, + 0.01131439208984375, + 0, + 0.006751265376806259, + 0, + 0, + 0, + -0.04573351889848709, + 0, + 0, + 0.04386100172996521, + 0.11277603358030319, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + -0.0003205372195225209, + 0, + 0, + 0, + -0.005409737583249807, + -0.009204162284731865, + 0, + 0, + 0, + 0.004804544150829315, + -0.005810749251395464, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.09645535796880722, + 0, + 0, + 0, + 0.032931435853242874, + 0, + 0, + 0, + 0, + 0.028524864464998245, + 0.09402520954608917, + 0, + 0.008998546749353409, + 0, + -0.03251685947179794, + 0, + 0, + 0, + 0, + 0, + 0, + 0.037343256175518036, + 0.0045840502716600895, + 0, + 0, + 0.009021798148751259, + 0, + -0.004217533860355616, + 0, + 0.0025705555453896523, + 0, + -0.035309672355651855, + -0.09942735731601715, + -0.004700342193245888, + 0, + 0, + 0, + 0.018288278952240944, + 0.0004021169152110815, + 0, + 0, + 0, + 0, + -0.005593586713075638, + 0.0018821493722498417, + 0, + 0, + 0, + 0, + -0.04561242088675499, + -0.01815006509423256, + 0.016583485528826714, + -0.020843051373958588, + 0, + 0, + 0, + -0.006372869946062565, + 0.04272369295358658, + 0, + 0.0013309348141774535, + 0, + 0, + 0, + -0.0031638317741453648, + 0, + 0.08714215457439423, + 0, + -0.005442100111395121, + -0.00039313771412707865, + 0.0014464370906352997, + 0, + 0, + 0, + 0, + -0.03132649511098862, + 0, + 0, + -0.007972904480993748, + 0.01753396727144718, + 0, + -0.0011563192820176482, + 0, + 0.0017362519865855575, + 0, + -0.0004587600124068558, + 0, + 0.0038881096988916397, + -0.008516360074281693, + 0, + -0.008183307014405727, + -0.010095844976603985, + 0, + 0.10722006857395172, + -0.002898464212194085, + 0, + 0.0012827662285417318, + 0.00004252225699019618, + 0, + 0.07567721605300903, + -0.030121177434921265, + 0, + -0.0010666534071788192, + 0.0006539365276694298, + 0, + 0, + -0.0011567147448658943, + -0.021622339263558388, + 0.028687214478850365, + 0, + 0, + 0.018764594569802284, + 0.010613140650093555, + 0.019510075449943542, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + -0.005288010463118553, + 0, + 0, + -0.016743114218115807, + -0.015873711556196213, + -0.009877816773951054, + 0.0003150522243231535, + 0.023689158260822296, + 0, + -0.00033418016391806304, + 0, + 0.04904749244451523, + 0, + 0, + 0, + 0.0006500506424345076, + 0.000622213410679251, + 0.00738720316439867, + 0, + 0.0012243357487022877, + 0.009066173806786537, + -0.12073952704668045, + 0.0010678119724616408, + 0.006296947598457336, + 0.002682592486962676, + -0.00444818427786231, + -0.0023324599023908377, + 0, + 0, + -0.5609893798828125, + -0.008780602365732193, + 0.015986066311597824, + 0, + -0.02213476411998272, + 0, + 0, + 0.0006705078994855285, + -0.0011221399763599038, + 0, + 0, + 0.025299811735749245, + 0, + -0.008218510076403618, + -0.0034782839938998222, + 0.0022423912305384874, + 0, + 0, + 0, + 0, + 0, + -0.05859537422657013, + 0.0421387143433094, + 0.26256099343299866, + 0, + 0, + 0, + 0, + 0, + -0.10330676287412643, + 0.012355834245681763, + 0.0013472040882334113, + 0, + 0.019914263859391212, + 0.005261276848614216, + 0.001149827498011291, + -0.03320133313536644, + 0, + 0, + 0, + 0, + 0.32198745012283325, + 0, + 0.05401667580008507, + 0.04610951617360115, + -0.2326284795999527, + 0, + 0.0000856258993735537, + 0, + 0.074106365442276, + 0, + 0.044469863176345825, + 0, + 0, + 0, + -0.006453251000493765, + 0, + -0.018431225791573524, + 0, + 0, + 0, + 0, + -0.005704954732209444, + 0, + 0, + 0, + 0, + 0, + 0, + 0.13457728922367096, + 0, + -0.029186677187681198, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + -0.022995056584477425, + 0, + 0, + 0, + -0.09004921466112137, + 0.24257110059261322, + 0.02852930873632431, + 0, + 0.021270141005516052, + -0.13564155995845795, + -0.03098711557686329, + 0, + 0, + 0, + 0.0486220121383667, + -0.001395023544318974, + 0, + 0.04929636791348457, + 0, + 0, + -0.13068373501300812, + 0, + 0, + 0.016955919563770294, + 0.03848254308104515, + 0, + -0.011160435155034065, + 0, + 0, + -0.0002991429646499455, + 0, + 0, + 0, + 0.01138608530163765, + -0.020150866359472275, + 0, + 0, + -0.007353566121309996, + -0.021389631554484367, + -0.042083244770765305, + 0.13586723804473877, + 0.005315479822456837, + 0, + 0.008157049305737019, + 0.022239860147237778, + 0, + 0, + 0.01896926946938038, + 0.0018052944215014577, + 0.0016496418975293636, + 0.0005593635141849518, + 0, + 0, + 0.07655386626720428, + 0, + 0, + 0, + 0.02781328558921814, + 0.04012482985854149, + 0, + 0, + 0, + 0.10631410032510757, + 0.03608629107475281, + 0, + -0.02651066705584526, + 0, + -0.0690990686416626, + 0, + 0, + 0, + 0, + 0, + 0, + 0.1022648885846138 + ], + "yaxis": "y" + } + ], + "layout": { + "legend": { + "tracegroupgap": 0 + }, + "shapes": [ + { + "line": { + "color": "gray", + "dash": "dot", + "width": 1 + }, + "type": "line", + "x0": -0.8653240203857422, + "x1": 0.31668567657470703, + "y0": -0.8653240203857422, + "y1": 0.31668567657470703 + } + ], + "template": { + "data": { + "bar": [ + { + "error_x": { + "color": "#2a3f5f" + }, + "error_y": { + "color": "#2a3f5f" + }, + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "bar" + } + ], + "barpolar": [ + { + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "barpolar" + } + ], + "carpet": [ + { + "aaxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "baxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "type": "carpet" + } + ], + "choropleth": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "choropleth" + } + ], + "contour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "contour" + } + ], + "contourcarpet": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "contourcarpet" + } + ], + "heatmap": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmap" + } + ], + "heatmapgl": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmapgl" + } + ], + "histogram": [ + { + "marker": { + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "histogram" + } + ], + "histogram2d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2d" + } + ], + "histogram2dcontour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2dcontour" + } + ], + "mesh3d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "mesh3d" + } + ], + "parcoords": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "parcoords" + } + ], + "pie": [ + { + "automargin": true, + "type": "pie" + } + ], + "scatter": [ + { + "fillpattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + }, + "type": "scatter" + } + ], + "scatter3d": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatter3d" + } + ], + "scattercarpet": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattercarpet" + } + ], + "scattergeo": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergeo" + } + ], + "scattergl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergl" + } + ], + "scattermapbox": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermapbox" + } + ], + "scatterpolar": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolar" + } + ], + "scatterpolargl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolargl" + } + ], + "scatterternary": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterternary" + } + ], + "surface": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "surface" + } + ], + "table": [ + { + "cells": { + "fill": { + "color": "#EBF0F8" + }, + "line": { + "color": "white" + } + }, + "header": { + "fill": { + "color": "#C8D4E3" + }, + "line": { + "color": "white" + } + }, + "type": "table" + } + ] + }, + "layout": { + "annotationdefaults": { + "arrowcolor": "#2a3f5f", + "arrowhead": 0, + "arrowwidth": 1 + }, + "autotypenumbers": "strict", + "coloraxis": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "colorscale": { + "diverging": [ + [ + 0, + "#8e0152" + ], + [ + 0.1, + "#c51b7d" + ], + [ + 0.2, + "#de77ae" + ], + [ + 0.3, + "#f1b6da" + ], + [ + 0.4, + "#fde0ef" + ], + [ + 0.5, + "#f7f7f7" + ], + [ + 0.6, + "#e6f5d0" + ], + [ + 0.7, + "#b8e186" + ], + [ + 0.8, + "#7fbc41" + ], + [ + 0.9, + "#4d9221" + ], + [ + 1, + "#276419" + ] + ], + "sequential": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "sequentialminus": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ] + }, + "colorway": [ + "#636efa", + "#EF553B", + "#00cc96", + "#ab63fa", + "#FFA15A", + "#19d3f3", + "#FF6692", + "#B6E880", + "#FF97FF", + "#FECB52" + ], + "font": { + "color": "#2a3f5f" + }, + "geo": { + "bgcolor": "white", + "lakecolor": "white", + "landcolor": "#E5ECF6", + "showlakes": true, + "showland": true, + "subunitcolor": "white" + }, + "hoverlabel": { + "align": "left" + }, + "hovermode": "closest", + "mapbox": { + "style": "light" + }, + "paper_bgcolor": "white", + "plot_bgcolor": "#E5ECF6", + "polar": { + "angularaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "radialaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "scene": { + "xaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "yaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "zaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + } + }, + "shapedefaults": { + "line": { + "color": "#2a3f5f" + } + }, + "ternary": { + "aaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "baxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "caxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "title": { + "x": 0.05 + }, + "xaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + }, + "yaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + } + } + }, + "title": { + "text": "Attribution vs Activation Patching Per SAE feature (L5 S2 Pos, all prompts)" + }, + "xaxis": { + "anchor": "y", + "domain": [ + 0, + 1 + ], + "title": { + "text": "Activation Patch" + } + }, + "yaxis": { + "anchor": "x", + "domain": [ + 0, + 1 + ], + "title": { + "text": "Attribution Patch" + } + } + } + }, + "text/html": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "fig = scatter(\n", + " y=sae_act_attr[:, s2_pos, all_live_features].flatten(), \n", + " x=causal_effects.flatten(),\n", + " title=\"Attribution vs Activation Patching Per SAE feature (L5 S2 Pos, all prompts)\",\n", + " xaxis=\"Activation Patch\",\n", + " yaxis=\"Attribution Patch\",\n", + " return_fig=True\n", + ")\n", + "fig.add_shape(\n", + " type='line',\n", + " x0=causal_effects.min(),\n", + " y0=causal_effects.min(),\n", + " x1=causal_effects.max(),\n", + " y1=causal_effects.max(),\n", + " line=dict(\n", + " color='gray',\n", + " width=1,\n", + " dash='dot'\n", + " )\n", + ")\n", + "fig.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "base", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/unit/test_hooked_sae.py b/tests/unit/test_hooked_sae.py new file mode 100644 index 000000000..de311772e --- /dev/null +++ b/tests/unit/test_hooked_sae.py @@ -0,0 +1,191 @@ +import einops +import pytest +import torch + +from transformer_lens import HookedSAE, HookedSAEConfig, HookedSAETransformer + +MODEL = "solu-1l" +prompt = "Hello World!" + + +class Counter: + def __init__(self): + self.count = 0 + + def inc(self, *args, **kwargs): + self.count += 1 + + +@pytest.fixture(scope="module") +def model(): + model = HookedSAETransformer.from_pretrained(MODEL) + yield model + model.reset_saes() + + +def get_sae_config(model, act_name): + site_to_size = { + "hook_z": model.cfg.d_head * model.cfg.n_heads, + "hook_mlp_out": model.cfg.d_model, + "hook_resid_pre": model.cfg.d_model, + "hook_post": model.cfg.d_mlp, + } + site = act_name.split(".")[-1] + d_in = site_to_size[site] + return HookedSAEConfig(d_in=d_in, d_sae=d_in * 2, hook_name=act_name) + + +@pytest.mark.parametrize( + "act_name", + [ + "blocks.0.attn.hook_z", + "blocks.0.hook_mlp_out", + "blocks.0.mlp.hook_post", + "blocks.0.hook_resid_pre", + ], +) +def test_forward_reconstructs_input(model, act_name): + """Verfiy that the HookedSAE returns an output with the same shape as the input activations.""" + sae_cfg = get_sae_config(model, act_name) + hooked_sae = HookedSAE(sae_cfg) + + _, cache = model.run_with_cache(prompt, names_filter=act_name) + x = cache[act_name] + + sae_output = hooked_sae(x) + assert sae_output.shape == x.shape + + +@pytest.mark.parametrize( + "act_name", + [ + "blocks.0.attn.hook_z", + "blocks.0.hook_mlp_out", + "blocks.0.mlp.hook_post", + "blocks.0.hook_resid_pre", + ], +) +def test_run_with_cache(model, act_name): + """Verifies that run_with_cache caches SAE activations""" + sae_cfg = get_sae_config(model, act_name) + hooked_sae = HookedSAE(sae_cfg) + + _, cache = model.run_with_cache(prompt, names_filter=act_name) + x = cache[act_name] + + sae_output, cache = hooked_sae.run_with_cache(x) + assert sae_output.shape == x.shape + + assert "hook_sae_input" in cache + assert "hook_sae_acts_pre" in cache + assert "hook_sae_acts_post" in cache + assert "hook_sae_recons" in cache + assert "hook_sae_output" in cache + + +@pytest.mark.parametrize( + "act_name", + [ + "blocks.0.attn.hook_z", + "blocks.0.hook_mlp_out", + "blocks.0.mlp.hook_post", + "blocks.0.hook_resid_pre", + ], +) +def test_run_with_hooks(model, act_name): + """Verifies that run_with_hooks works with SAE activations""" + c = Counter() + sae_cfg = get_sae_config(model, act_name) + hooked_sae = HookedSAE(sae_cfg) + + _, cache = model.run_with_cache(prompt, names_filter=act_name) + x = cache[act_name] + + sae_hooks = [ + "hook_sae_input", + "hook_sae_acts_pre", + "hook_sae_acts_post", + "hook_sae_recons", + "hook_sae_output", + ] + + sae_output = hooked_sae.run_with_hooks( + x, fwd_hooks=[(sae_hook_name, c.inc) for sae_hook_name in sae_hooks] + ) + assert sae_output.shape == x.shape + + assert c.count == len(sae_hooks) + + +@pytest.mark.parametrize( + "act_name", + [ + "blocks.0.attn.hook_z", + "blocks.0.hook_mlp_out", + "blocks.0.mlp.hook_post", + "blocks.0.hook_resid_pre", + ], +) +def test_error_term(model, act_name): + """Verifies that that if we use error_terms, HookedSAE returns an output that is equal to the input activations.""" + sae_cfg = get_sae_config(model, act_name) + sae_cfg.use_error_term = True + hooked_sae = HookedSAE(sae_cfg) + + _, cache = model.run_with_cache(prompt, names_filter=act_name) + x = cache[act_name] + + sae_output = hooked_sae(x) + assert sae_output.shape == x.shape + assert torch.allclose(sae_output, x, atol=1e-6) + + +# %% +@pytest.mark.parametrize( + "act_name", + [ + "blocks.0.attn.hook_z", + "blocks.0.hook_mlp_out", + "blocks.0.mlp.hook_post", + "blocks.0.hook_resid_pre", + ], +) +def test_feature_grads_with_error_term(model, act_name): + """Verifies that pytorch backward computes the correct feature gradients when using error_terms. Motivated by the need to compute feature gradients for attribution patching.""" + + # Load SAE + sae_cfg = get_sae_config(model, act_name) + sae_cfg.use_error_term = True + hooked_sae = HookedSAE(sae_cfg) + + # Get input activations + _, cache = model.run_with_cache(prompt, names_filter=act_name) + x = cache[act_name] + + # Cache gradients with respect to feature acts + hooked_sae.reset_hooks() + grad_cache = {} + + def backward_cache_hook(act, hook): + grad_cache[hook.name] = act.detach() + + hooked_sae.add_hook("hook_sae_acts_post", backward_cache_hook, "bwd") + hooked_sae.add_hook("hook_sae_output", backward_cache_hook, "bwd") + + sae_output = hooked_sae(x) + assert torch.allclose(sae_output, x, atol=1e-6) + value = sae_output.sum() + value.backward() + hooked_sae.reset_hooks() + + # Compute gradient analytically + if act_name.endswith("hook_z"): + reshaped_output_grad = einops.rearrange( + grad_cache["hook_sae_output"], "... n_heads d_head -> ... (n_heads d_head)" + ) + analytic_grad = reshaped_output_grad @ hooked_sae.W_dec.T + else: + analytic_grad = grad_cache["hook_sae_output"] @ hooked_sae.W_dec.T + + # Compare analytic gradient with pytorch computed gradient + assert torch.allclose(grad_cache["hook_sae_acts_post"], analytic_grad, atol=1e-6) diff --git a/tests/unit/test_hooked_sae_transformer.py b/tests/unit/test_hooked_sae_transformer.py new file mode 100644 index 000000000..bfb428c8e --- /dev/null +++ b/tests/unit/test_hooked_sae_transformer.py @@ -0,0 +1,515 @@ +import pytest +import torch + +from transformer_lens import ( + HookedSAE, + HookedSAEConfig, + HookedSAETransformer, + HookedTransformer, +) +from transformer_lens.ActivationCache import ActivationCache +from transformer_lens.hook_points import HookPoint # Hooking utilities +from transformer_lens.HookedSAETransformer import get_deep_attr + +MODEL = "solu-1l" +prompt = "Hello World!" + + +class Counter: + def __init__(self): + self.count = 0 + + def inc(self, *args, **kwargs): + self.count += 1 + + +@pytest.fixture(scope="module") +def original_logits(): + original_model = HookedTransformer.from_pretrained(MODEL) + return original_model(prompt) + + +@pytest.fixture(scope="module") +def model(): + model = HookedSAETransformer.from_pretrained(MODEL) + yield model + model.reset_saes() + + +def get_sae_config(model, act_name): + site_to_size = { + "hook_z": model.cfg.d_head * model.cfg.n_heads, + "hook_mlp_out": model.cfg.d_model, + "hook_resid_pre": model.cfg.d_model, + "hook_post": model.cfg.d_mlp, + } + site = act_name.split(".")[-1] + d_in = site_to_size[site] + return HookedSAEConfig(d_in=d_in, d_sae=d_in * 2, hook_name=act_name) + + +def test_model_with_no_saes_matches_original_model(model, original_logits): + """Verifies that HookedSAETransformer behaves like a normal HookedTransformer model when no SAEs are attached.""" + assert len(model.acts_to_saes) == 0 + logits = model(prompt) + assert torch.allclose(original_logits, logits) + + +@pytest.mark.parametrize( + "act_name", + [ + "blocks.0.attn.hook_z", + "blocks.0.hook_mlp_out", + "blocks.0.mlp.hook_post", + "blocks.0.hook_resid_pre", + ], +) +def test_model_with_saes_does_not_match_original_model(model, act_name, original_logits): + """Verifies that the attached (and turned on) SAEs actually affect the models output logits""" + assert len(model.acts_to_saes) == 0 + sae_cfg = get_sae_config(model, act_name) + hooked_sae = HookedSAE(sae_cfg) + model.add_sae(hooked_sae) + assert len(model.acts_to_saes) == 1 + logits_with_saes = model(prompt) + assert not torch.allclose(original_logits, logits_with_saes) + model.reset_saes() + + +@pytest.mark.parametrize( + "act_name", + [ + "blocks.0.attn.hook_z", + "blocks.0.hook_mlp_out", + "blocks.0.mlp.hook_post", + "blocks.0.hook_resid_pre", + ], +) +def test_add_sae(model, act_name): + """Verifies that add_sae correctly updates the model's acts_to_saes dictionary and replaces the HookPoint.""" + sae_cfg = get_sae_config(model, act_name) + hooked_sae = HookedSAE(sae_cfg) + model.add_sae(hooked_sae) + assert len(model.acts_to_saes) == 1 + assert model.acts_to_saes[act_name] == hooked_sae + assert get_deep_attr(model, act_name) == hooked_sae + model.reset_saes() + + +@pytest.mark.parametrize( + "act_name", + [ + "blocks.0.attn.hook_z", + "blocks.0.hook_mlp_out", + "blocks.0.mlp.hook_post", + "blocks.0.hook_resid_pre", + ], +) +def test_add_sae_overwrites_prev_sae(model, act_name): + """Verifies that add_sae correctly updates the model's acts_to_saes dictionary and replaces the HookPoint.""" + prev_sae_cfg = get_sae_config(model, act_name) + prev_hooked_sae = HookedSAE(prev_sae_cfg) + model.add_sae(prev_hooked_sae) + assert len(model.acts_to_saes) == 1 + assert model.acts_to_saes[act_name] == prev_hooked_sae + assert get_deep_attr(model, act_name) == prev_hooked_sae + + sae_cfg = get_sae_config(model, act_name) + hooked_sae = HookedSAE(sae_cfg) + model.add_sae(hooked_sae) + assert len(model.acts_to_saes) == 1 + assert model.acts_to_saes[act_name] == hooked_sae + assert get_deep_attr(model, act_name) == hooked_sae + model.reset_saes() + + +@pytest.mark.parametrize( + "act_name", + [ + "blocks.0.attn.hook_z", + "blocks.0.hook_mlp_out", + "blocks.0.mlp.hook_post", + "blocks.0.hook_resid_pre", + ], +) +def test_reset_sae_removes_sae_by_default(model, act_name): + """Verifies that reset_sae correctly removes the SAE from the model's acts_to_saes dictionary and replaces the HookedSAE with a HookPoint.""" + sae_cfg = get_sae_config(model, act_name) + hooked_sae = HookedSAE(sae_cfg) + model.add_sae(hooked_sae) + assert len(model.acts_to_saes) == 1 + assert model.acts_to_saes[act_name] == hooked_sae + assert get_deep_attr(model, act_name) == hooked_sae + model._reset_sae(act_name) + assert len(model.acts_to_saes) == 0 + assert isinstance(get_deep_attr(model, act_name), HookPoint) + model.reset_saes() + + +@pytest.mark.parametrize( + "act_name", + [ + "blocks.0.attn.hook_z", + "blocks.0.hook_mlp_out", + "blocks.0.mlp.hook_post", + "blocks.0.hook_resid_pre", + ], +) +def test_reset_sae_replaces_sae(model, act_name): + """Verifies that reset_sae correctly removes the SAE from the model's acts_to_saes dictionary and replaces the HookedSAE with a HookPoint.""" + sae_cfg = get_sae_config(model, act_name) + hooked_sae = HookedSAE(sae_cfg) + + prev_sae_cfg = get_sae_config(model, act_name) + prev_sae = HookedSAE(prev_sae_cfg) + + model.add_sae(hooked_sae) + assert len(model.acts_to_saes) == 1 + assert model.acts_to_saes[act_name] == hooked_sae + assert get_deep_attr(model, act_name) == hooked_sae + model._reset_sae(act_name, prev_sae) + assert len(model.acts_to_saes) == 1 + assert get_deep_attr(model, act_name) == prev_sae + model.reset_saes() + + +@pytest.mark.parametrize( + "act_names", + [ + ["blocks.0.attn.hook_z"], + ["blocks.0.hook_mlp_out"], + ["blocks.0.mlp.hook_post"], + ["blocks.0.hook_resid_pre"], + [ + "blocks.0.attn.hook_z", + "blocks.0.hook_mlp_out", + "blocks.0.mlp.hook_post", + "blocks.0.hook_resid_pre", + ], + ], +) +def test_reset_saes_removes_all_saes_by_default(model, act_names): + """Verifies that reset_saes correctly removes all SAEs from the model's acts_to_saes dictionary and replaces the HookedSAEs with HookPoints.""" + sae_cfgs = [get_sae_config(model, act_name) for act_name in act_names] + hooked_saes = [HookedSAE(sae_cfg) for sae_cfg in sae_cfgs] + for hooked_sae in hooked_saes: + model.add_sae(hooked_sae) + assert len(model.acts_to_saes) == len(act_names) + for act_name, hooked_sae in zip(act_names, hooked_saes): + assert model.acts_to_saes[act_name] == hooked_sae + assert get_deep_attr(model, act_name) == hooked_sae + model.reset_saes() + assert len(model.acts_to_saes) == 0 + for act_name in act_names: + assert isinstance(get_deep_attr(model, act_name), HookPoint) + model.reset_saes() + + +@pytest.mark.parametrize( + "act_names", + [ + ["blocks.0.attn.hook_z"], + ["blocks.0.hook_mlp_out"], + ["blocks.0.mlp.hook_post"], + ["blocks.0.hook_resid_pre"], + [ + "blocks.0.attn.hook_z", + "blocks.0.hook_mlp_out", + "blocks.0.mlp.hook_post", + "blocks.0.hook_resid_pre", + ], + ], +) +def test_reset_saes_replaces_saes(model, act_names): + """Verifies that reset_saes correctly removes all SAEs from the model's acts_to_saes dictionary and replaces the HookedSAEs with HookPoints.""" + sae_cfgs = [get_sae_config(model, act_name) for act_name in act_names] + hooked_saes = [HookedSAE(sae_cfg) for sae_cfg in sae_cfgs] + for hooked_sae in hooked_saes: + model.add_sae(hooked_sae) + + prev_sae_cfgs = [get_sae_config(model, act_name) for act_name in act_names] + prev_hooked_saes = [HookedSAE(prev_sae_cfg) for prev_sae_cfg in prev_sae_cfgs] + + assert len(model.acts_to_saes) == len(act_names) + for act_name, hooked_sae in zip(act_names, hooked_saes): + assert model.acts_to_saes[act_name] == hooked_sae + assert get_deep_attr(model, act_name) == hooked_sae + model.reset_saes(act_names, prev_hooked_saes) + assert len(model.acts_to_saes) == len(prev_hooked_saes) + for act_name, prev_hooked_sae in zip(act_names, prev_hooked_saes): + assert get_deep_attr(model, act_name) == prev_hooked_sae + model.reset_saes() + + +@pytest.mark.parametrize( + "act_names", + [ + ["blocks.0.attn.hook_z"], + ["blocks.0.hook_mlp_out"], + ["blocks.0.mlp.hook_post"], + ["blocks.0.hook_resid_pre"], + [ + "blocks.0.attn.hook_z", + "blocks.0.hook_mlp_out", + "blocks.0.mlp.hook_post", + "blocks.0.hook_resid_pre", + ], + ], +) +def test_saes_context_manager_removes_saes_after(model, act_names): + """Verifies that the model.saes context manager successfully adds the SAEs for the specified activation name in the context manager and resets off after the context manager exits.""" + sae_cfgs = [get_sae_config(model, act_name) for act_name in act_names] + hooked_saes = [HookedSAE(sae_cfg) for sae_cfg in sae_cfgs] + assert len(model.acts_to_saes) == 0 + for act_name in act_names: + assert isinstance(get_deep_attr(model, act_name), HookPoint) + with model.saes(saes=hooked_saes): + for act_name, hooked_sae in zip(act_names, hooked_saes): + assert model.acts_to_saes[act_name] == hooked_sae + assert isinstance(get_deep_attr(model, act_name), HookedSAE) + assert get_deep_attr(model, act_name) == hooked_sae + model.forward(prompt) + assert len(model.acts_to_saes) == 0 + for act_name in act_names: + assert isinstance(get_deep_attr(model, act_name), HookPoint) + model.reset_saes() + + +@pytest.mark.parametrize( + "act_names", + [ + ["blocks.0.attn.hook_z"], + ["blocks.0.hook_mlp_out"], + ["blocks.0.mlp.hook_post"], + ["blocks.0.hook_resid_pre"], + [ + "blocks.0.attn.hook_z", + "blocks.0.hook_mlp_out", + "blocks.0.mlp.hook_post", + "blocks.0.hook_resid_pre", + ], + ], +) +def test_saes_context_manager_restores_previous_sae_state(model, act_names): + """Verifies that the model.saes context manager successfully adds the SAEs for the specified activation name in the context manager and resets off after the context manager exits.""" + # First add SAEs statefully + prev_sae_cfgs = [get_sae_config(model, act_name) for act_name in act_names] + prev_hooked_saes = [HookedSAE(sae_cfg) for sae_cfg in prev_sae_cfgs] + for act_name, prev_hooked_sae in zip(act_names, prev_hooked_saes): + model.add_sae(prev_hooked_sae) + assert get_deep_attr(model, act_name) == prev_hooked_sae + assert len(model.acts_to_saes) == len(prev_hooked_saes) + + # Now temporarily run with new SAEs + sae_cfgs = [get_sae_config(model, act_name) for act_name in act_names] + hooked_saes = [HookedSAE(sae_cfg) for sae_cfg in sae_cfgs] + with model.saes(saes=hooked_saes): + for act_name, hooked_sae in zip(act_names, hooked_saes): + assert model.acts_to_saes[act_name] == hooked_sae + assert isinstance(get_deep_attr(model, act_name), HookedSAE) + assert get_deep_attr(model, act_name) == hooked_sae + model.forward(prompt) + + # Check that the previously attached SAEs have been restored + assert len(model.acts_to_saes) == len(prev_hooked_saes) + for act_name, prev_hooked_sae in zip(act_names, prev_hooked_saes): + assert isinstance(get_deep_attr(model, act_name), HookedSAE) + assert get_deep_attr(model, act_name) == prev_hooked_sae + model.reset_saes() + + +@pytest.mark.parametrize( + "act_names", + [ + ["blocks.0.attn.hook_z"], + ["blocks.0.hook_mlp_out"], + ["blocks.0.mlp.hook_post"], + ["blocks.0.hook_resid_pre"], + [ + "blocks.0.attn.hook_z", + "blocks.0.hook_mlp_out", + "blocks.0.mlp.hook_post", + "blocks.0.hook_resid_pre", + ], + ], +) +def test_saes_context_manager_run_with_cache(model, act_names): + """Verifies that the model.run_with_cache method works correctly in the context manager.""" + sae_cfgs = [get_sae_config(model, act_name) for act_name in act_names] + hooked_saes = [HookedSAE(sae_cfg) for sae_cfg in sae_cfgs] + assert len(model.acts_to_saes) == 0 + for act_name in act_names: + assert isinstance(get_deep_attr(model, act_name), HookPoint) + with model.saes(saes=hooked_saes): + for act_name, hooked_sae in zip(act_names, hooked_saes): + assert model.acts_to_saes[act_name] == hooked_sae + assert isinstance(get_deep_attr(model, act_name), HookedSAE) + assert get_deep_attr(model, act_name) == hooked_sae + model.run_with_cache(prompt) + assert len(model.acts_to_saes) == 0 + for act_name in act_names: + assert isinstance(get_deep_attr(model, act_name), HookPoint) + model.reset_saes() + + +@pytest.mark.parametrize( + "act_names", + [ + ["blocks.0.attn.hook_z"], + ["blocks.0.hook_mlp_out"], + ["blocks.0.mlp.hook_post"], + ["blocks.0.hook_resid_pre"], + [ + "blocks.0.attn.hook_z", + "blocks.0.hook_mlp_out", + "blocks.0.mlp.hook_post", + "blocks.0.hook_resid_pre", + ], + ], +) +def test_run_with_saes(model, act_names, original_logits): + """Verifies that the model.run_with_saes method works correctly. The logits with SAEs should be different from the original logits, but the SAE should be removed immediately after the forward pass.""" + sae_cfgs = [get_sae_config(model, act_name) for act_name in act_names] + hooked_saes = [HookedSAE(sae_cfg) for sae_cfg in sae_cfgs] + assert len(model.acts_to_saes) == 0 + logits_with_saes = model.run_with_saes(prompt, saes=hooked_saes) + assert not torch.allclose(logits_with_saes, original_logits) + assert len(model.acts_to_saes) == 0 + for act_name in act_names: + assert isinstance(get_deep_attr(model, act_name), HookPoint) + model.reset_saes() + + +@pytest.mark.parametrize( + "act_names", + [ + ["blocks.0.attn.hook_z"], + ["blocks.0.hook_mlp_out"], + ["blocks.0.mlp.hook_post"], + ["blocks.0.hook_resid_pre"], + [ + "blocks.0.attn.hook_z", + "blocks.0.hook_mlp_out", + "blocks.0.mlp.hook_post", + "blocks.0.hook_resid_pre", + ], + ], +) +def test_run_with_cache(model, act_names, original_logits): + """Verifies that the model.run_with_cache method works correctly. The logits with SAEs should be different from the original logits and the cache should contain SAE activations for the attached SAE.""" + sae_cfgs = [get_sae_config(model, act_name) for act_name in act_names] + hooked_saes = [HookedSAE(sae_cfg) for sae_cfg in sae_cfgs] + for hooked_sae in hooked_saes: + model.add_sae(hooked_sae) + assert len(model.acts_to_saes) == len(hooked_saes) + logits_with_saes, cache = model.run_with_cache(prompt) + assert not torch.allclose(logits_with_saes, original_logits) + assert isinstance(cache, ActivationCache) + for act_name, hooked_sae in zip(act_names, hooked_saes): + assert act_name + ".hook_sae_acts_post" in cache + assert isinstance(get_deep_attr(model, act_name), HookedSAE) + assert get_deep_attr(model, act_name) == hooked_sae + model.reset_saes() + + +@pytest.mark.parametrize( + "act_names", + [ + ["blocks.0.attn.hook_z"], + ["blocks.0.hook_mlp_out"], + ["blocks.0.mlp.hook_post"], + ["blocks.0.hook_resid_pre"], + [ + "blocks.0.attn.hook_z", + "blocks.0.hook_mlp_out", + "blocks.0.mlp.hook_post", + "blocks.0.hook_resid_pre", + ], + ], +) +def test_run_with_cache_with_saes(model, act_names, original_logits): + """Verifies that the model.run_with_cache_with_saes method works correctly. The logits with SAEs should be different from the original logits and the cache should contain SAE activations for the attached SAE.""" + sae_cfgs = [get_sae_config(model, act_name) for act_name in act_names] + hooked_saes = [HookedSAE(sae_cfg) for sae_cfg in sae_cfgs] + logits_with_saes, cache = model.run_with_cache_with_saes(prompt, saes=hooked_saes) + assert not torch.allclose(logits_with_saes, original_logits) + assert isinstance(cache, ActivationCache) + + assert len(model.acts_to_saes) == 0 + for act_name, hooked_sae in zip(act_names, hooked_saes): + assert act_name + ".hook_sae_acts_post" in cache + assert isinstance(get_deep_attr(model, act_name), HookPoint) + model.reset_saes() + + +@pytest.mark.parametrize( + "act_names", + [ + ["blocks.0.attn.hook_z"], + ["blocks.0.hook_mlp_out"], + ["blocks.0.mlp.hook_post"], + ["blocks.0.hook_resid_pre"], + [ + "blocks.0.attn.hook_z", + "blocks.0.hook_mlp_out", + "blocks.0.mlp.hook_post", + "blocks.0.hook_resid_pre", + ], + ], +) +def test_run_with_hooks(model, act_names, original_logits): + """Verifies that the model.run_with_hooks method works correctly when SAEs are attached. The count should be incremented by 1 when the hooked SAE is called, and the SAE should stay attached after the forward pass""" + c = Counter() + sae_cfgs = [get_sae_config(model, act_name) for act_name in act_names] + hooked_saes = [HookedSAE(sae_cfg) for sae_cfg in sae_cfgs] + + for hooked_sae in hooked_saes: + model.add_sae(hooked_sae) + + logits_with_saes = model.run_with_hooks( + prompt, fwd_hooks=[(act_name + ".hook_sae_acts_post", c.inc) for act_name in act_names] + ) + assert not torch.allclose(logits_with_saes, original_logits) + + for act_name, hooked_sae in zip(act_names, hooked_saes): + assert isinstance(get_deep_attr(model, act_name), HookedSAE) + assert get_deep_attr(model, act_name) == hooked_sae + assert c.count == len(act_names) + model.reset_saes() + model.remove_all_hook_fns(including_permanent=True) + + +@pytest.mark.parametrize( + "act_names", + [ + ["blocks.0.attn.hook_z"], + ["blocks.0.hook_mlp_out"], + ["blocks.0.mlp.hook_post"], + ["blocks.0.hook_resid_pre"], + [ + "blocks.0.attn.hook_z", + "blocks.0.hook_mlp_out", + "blocks.0.mlp.hook_post", + "blocks.0.hook_resid_pre", + ], + ], +) +def test_run_with_hooks_with_saes(model, act_names, original_logits): + """Verifies that the model.run_with_hooks_with_saes method works correctly when SAEs are attached. The count should be incremented by 1 when the hooked SAE is called, but the SAE should be removed immediately after the forward pass.""" + c = Counter() + sae_cfgs = [get_sae_config(model, act_name) for act_name in act_names] + hooked_saes = [HookedSAE(sae_cfg) for sae_cfg in sae_cfgs] + + logits_with_saes = model.run_with_hooks_with_saes( + prompt, + saes=hooked_saes, + fwd_hooks=[(act_name + ".hook_sae_acts_post", c.inc) for act_name in act_names], + ) + assert not torch.allclose(logits_with_saes, original_logits) + assert c.count == len(act_names) + + assert len(model.acts_to_saes) == 0 + for act_name in act_names: + assert isinstance(get_deep_attr(model, act_name), HookPoint) + model.reset_saes() + model.remove_all_hook_fns(including_permanent=True) diff --git a/transformer_lens/HookedSAE.py b/transformer_lens/HookedSAE.py new file mode 100644 index 000000000..df9b29d05 --- /dev/null +++ b/transformer_lens/HookedSAE.py @@ -0,0 +1,118 @@ +from typing import Dict, Union + +import einops +import torch +import torch.nn.functional as F +from jaxtyping import Float +from torch import nn + +from transformer_lens.hook_points import ( # Hooking utilities + HookedRootModule, + HookPoint, +) +from transformer_lens.HookedSAEConfig import HookedSAEConfig + + +class HookedSAE(HookedRootModule): + """Hooked SAE. + + Implements a standard SAE with a TransformerLens hooks for SAE activations + + Designed for inference / analysis, not training. For training, see Joseph Bloom's SAELens (https://github.com/jbloomAus/SAELens) + + Note that HookedSAETransformer is fairly modular, and doesn't make strong assumptions about the architecture of the SAEs that get attached. We provide HookedSAE as a useful default class, but if you want to eg experiment with other SAE architectures, you can just copy the HookedSAE code into a notebook, edit it, and add instances of the new SAE class to a HookedSAETransformer (e.g. with HookedSAETransformer.add_sae(sae)) + """ + + def __init__(self, cfg: Union[HookedSAEConfig, Dict]): + super().__init__() + if isinstance(cfg, Dict): + cfg = HookedSAEConfig(**cfg) + elif isinstance(cfg, str): + raise ValueError("Please pass in a config dictionary or HookedSAEConfig object.") + self.cfg = cfg + + self.W_enc = nn.Parameter( + torch.nn.init.kaiming_uniform_( + torch.empty(self.cfg.d_in, self.cfg.d_sae, dtype=self.cfg.dtype) + ) + ) + self.W_dec = nn.Parameter( + torch.nn.init.kaiming_uniform_( + torch.empty(self.cfg.d_sae, self.cfg.d_in, dtype=self.cfg.dtype) + ) + ) + self.b_enc = nn.Parameter(torch.zeros(self.cfg.d_sae, dtype=self.cfg.dtype)) + self.b_dec = nn.Parameter(torch.zeros(self.cfg.d_in, dtype=self.cfg.dtype)) + + self.hook_sae_input = HookPoint() + self.hook_sae_acts_pre = HookPoint() + self.hook_sae_acts_post = HookPoint() + self.hook_sae_recons = HookPoint() + self.hook_sae_error = HookPoint() + self.hook_sae_output = HookPoint() + + self.to(self.cfg.device) + self.setup() + + def forward(self, input: Float[torch.Tensor, "... d_in"]) -> Float[torch.Tensor, "... d_in"]: + """SAE Forward Pass. + + Args: + input: The input tensor of activations to the SAE. Shape [..., d_in]. + Also supports hook_z activations of shape [..., n_heads, d_head], where n_heads * d_head = d_in, for attention output (hook_z) SAEs. + + Returns: + output: The reconstructed output tensor from the SAE, with the error term optionally added. Same shape as input (eg [..., d_in]) + """ + self.hook_sae_input(input) + if input.shape[-1] == self.cfg.d_in: + x = input + else: + # Assume this this is an attention output (hook_z) SAE + assert self.cfg.hook_name.endswith( + "_z" + ), f"You passed in an input shape {input.shape} does not match SAE input size {self.cfg.d_in} for hook_name {self.cfg.hook_name}. This is only supported for attn output (hook_z) SAEs." + x = einops.rearrange(input, "... n_heads d_head -> ... (n_heads d_head)") + assert ( + x.shape[-1] == self.cfg.d_in + ), f"Input shape {x.shape} does not match SAE input size {self.cfg.d_in}" + + x_cent = x - self.b_dec + # WARNING: if editing this block of code, also edit the error computation inside `if self.cfg.use_error_term` + sae_acts_pre = self.hook_sae_acts_pre( + einops.einsum(x_cent, self.W_enc, "... d_in, d_in d_sae -> ... d_sae") + + self.b_enc # [..., d_sae] + ) + sae_acts_post = self.hook_sae_acts_post(F.relu(sae_acts_pre)) # [..., d_sae] + x_reconstruct = self.hook_sae_recons( + ( + einops.einsum(sae_acts_post, self.W_dec, "... d_sae, d_sae d_in -> ... d_in") + + self.b_dec + ).reshape(input.shape) + ) + # END WARNING + + if self.cfg.use_error_term: + with torch.no_grad(): + # Recompute everything without hooks to get true error term + # Otherwise, the output with error term will always equal input, even for causal interventions that affect x_reconstruct + # This is in a no_grad context to detach the error, so we can compute SAE feature gradients (eg for attribution patching). See A.3 in https://arxiv.org/pdf/2403.19647.pdf for more detail + # NOTE: we can't just use `sae_error = input - x_reconstruct.detach()` or something simpler, since this would mean intervening on features would mean ablating features still results in perfect reconstruction. + sae_acts_pre_clean = ( + einops.einsum(x_cent, self.W_enc, "... d_in, d_in d_sae -> ... d_sae") + + self.b_enc + ) # [..., d_sae] + sae_acts_post_clean = F.relu(sae_acts_pre_clean) + x_reconstruct_clean = ( + einops.einsum( + sae_acts_post_clean, + self.W_dec, + "... d_sae, d_sae d_in -> ... d_in", + ) + + self.b_dec + ).reshape(input.shape) + + sae_error = self.hook_sae_error(input - x_reconstruct_clean) + return self.hook_sae_output(x_reconstruct + sae_error) + + return self.hook_sae_output(x_reconstruct) diff --git a/transformer_lens/HookedSAEConfig.py b/transformer_lens/HookedSAEConfig.py new file mode 100644 index 000000000..2892329e4 --- /dev/null +++ b/transformer_lens/HookedSAEConfig.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import pprint +import random +from dataclasses import dataclass +from typing import Any, Dict, Optional + +import numpy as np +import torch + +from transformer_lens import utils + + +@dataclass +class HookedSAEConfig: + """ + Configuration class to store the configuration of a HookedSAE model. + + Args: + d_sae (int): The size of the dictionary. + d_in (int): The dimension of the input activations. + hook_name (str): The hook name of the activation the SAE was trained on (eg. blocks.0.attn.hook_z) + use_error_term (bool): Whether to use the error term in the loss function. Defaults to False. + dtype (torch.dtype, *optional*): The SAE's dtype. Defaults to torch.float32. + seed (int, *optional*): The seed to use for the SAE. + Used to set sources of randomness (Python, PyTorch and + NumPy) and to initialize weights. Defaults to None. We recommend setting a seed, so your experiments are reproducible. + device(str): The device to use for the SAE. Defaults to 'cuda' if + available, else 'cpu'. + """ + + d_sae: int + d_in: int + hook_name: str + use_error_term: bool = False + dtype: torch.dtype = torch.float32 + seed: Optional[int] = None + device: Optional[str] = None + + def __post_init__(self): + if self.seed is not None: + self.set_seed_everywhere(self.seed) + + if self.device is None: + self.device = utils.get_device() + + @classmethod + def from_dict(cls, config_dict: Dict[str, Any]) -> HookedSAEConfig: + """ + Instantiates a `HookedSAEConfig` from a Python dictionary of + parameters. + """ + return cls(**config_dict) + + def to_dict(self): + return self.__dict__ + + def __repr__(self): + return "HookedSAEConfig:\n" + pprint.pformat(self.to_dict()) + + def set_seed_everywhere(self, seed: int): + torch.manual_seed(seed) + random.seed(seed) + np.random.seed(seed) diff --git a/transformer_lens/HookedSAETransformer.py b/transformer_lens/HookedSAETransformer.py new file mode 100644 index 000000000..47e88ebb9 --- /dev/null +++ b/transformer_lens/HookedSAETransformer.py @@ -0,0 +1,290 @@ +import logging +from contextlib import contextmanager +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import torch +from jaxtyping import Float + +from transformer_lens.ActivationCache import ActivationCache +from transformer_lens.hook_points import HookPoint # Hooking utilities +from transformer_lens.HookedSAE import HookedSAE +from transformer_lens.HookedTransformer import HookedTransformer + +SingleLoss = Float[torch.Tensor, ""] # Type alias for a single element tensor +LossPerToken = Float[torch.Tensor, "batch pos-1"] +Loss = Union[SingleLoss, LossPerToken] + + +def get_deep_attr(obj: Any, path: str): + """Helper function to get a nested attribute from a object. + In practice used to access HookedTransformer HookPoints (eg model.blocks[0].attn.hook_z) + + Args: + obj: Any object. In practice, this is a HookedTransformer (or subclass) + path: str. The path to the attribute you want to access. (eg "blocks.0.attn.hook_z") + + returns: + Any. The attribute at the end of the path + """ + parts = path.split(".") + # Navigate to the last component in the path + for part in parts: + if part.isdigit(): # This is a list index + obj = obj[int(part)] + else: # This is an attribute + obj = getattr(obj, part) + return obj + + +def set_deep_attr(obj: Any, path: str, value: Any): + """Helper function to change the value of a nested attribute from a object. + In practice used to swap HookedTransformer HookPoints (eg model.blocks[0].attn.hook_z) with HookedSAEs and vice versa + + Args: + obj: Any object. In practice, this is a HookedTransformer (or subclass) + path: str. The path to the attribute you want to access. (eg "blocks.0.attn.hook_z") + value: Any. The value you want to set the attribute to (eg a HookedSAE object) + """ + parts = path.split(".") + # Navigate to the last component in the path + for part in parts[:-1]: + if part.isdigit(): # This is a list index + obj = obj[int(part)] + else: # This is an attribute + obj = getattr(obj, part) + # Set the value on the final attribute + setattr(obj, parts[-1], value) + + +class HookedSAETransformer(HookedTransformer): + def __init__( + self, + *model_args, + **model_kwargs, + ): + """Model initialization. Just HookedTransformer init, but adds a dictionary to keep track of attached SAEs. + + Note that if you want to load the model from pretrained weights, you should use + :meth:`from_pretrained` instead. + + Args: + *model_args: Positional arguments for HookedTransformer initialization + **model_kwargs: Keyword arguments for HookedTransformer initialization + """ + super().__init__(*model_args, **model_kwargs) + self.acts_to_saes: Dict[str, HookedSAE] = {} + + def add_sae(self, sae: HookedSAE): + """Attaches an SAE to the model + + WARNING: This sae will be permanantly attached until you remove it with reset_saes. This function will also overwrite any existing SAE attached to the same hook point. + + Args: + sae: HookedSAE. The SAE to attach to the model + """ + act_name = sae.cfg.hook_name + if (act_name not in self.acts_to_saes) and (act_name not in self.hook_dict): + logging.warning( + f"No hook found for {act_name}. Skipping. Check model.hook_dict for available hooks." + ) + return + + self.acts_to_saes[act_name] = sae + set_deep_attr(self, act_name, sae) + self.setup() + + def _reset_sae(self, act_name: str, prev_sae: Optional[HookedSAE] = None): + """Resets an SAE that was attached to the model + + By default will remove the SAE from that hook_point. + If prev_sae is provided, will replace the current SAE with the provided one. + This is mainly used to restore previously attached SAEs after temporarily running with different SAEs (eg with run_with_saes) + + Args: + act_name: str. The hook_name of the SAE to reset + prev_sae: Optional[HookedSAE]. The SAE to replace the current one with. If None, will just remove the SAE from this hook point. Defaults to None + """ + if act_name not in self.acts_to_saes: + logging.warning(f"No SAE is attached to {act_name}. There's nothing to reset.") + return + + if prev_sae: + set_deep_attr(self, act_name, prev_sae) + self.acts_to_saes[act_name] = prev_sae + else: + set_deep_attr(self, act_name, HookPoint()) + del self.acts_to_saes[act_name] + + def reset_saes( + self, + act_names: Optional[Union[str, List[str]]] = None, + prev_saes: Optional[List[Union[HookedSAE, None]]] = None, + ): + """Reset the SAEs attached to the model + + If act_names are provided will just reset SAEs attached to those hooks. Otherwise will reset all SAEs attached to the model. + Optionally can provide a list of prev_saes to reset to. This is mainly used to restore previously attached SAEs after temporarily running with different SAEs (eg with run_with_saes). + + Args: + act_names (Optional[Union[str, List[str]]): The act_names of the SAEs to reset. If None, will reset all SAEs attached to the model. Defaults to None. + prev_saes (Optional[List[Union[HookedSAE, None]]]): List of SAEs to replace the current ones with. If None, will just remove the SAEs. Defaults to None. + """ + if isinstance(act_names, str): + act_names = [act_names] + elif act_names is None: + act_names = list(self.acts_to_saes.keys()) + + if prev_saes: + assert len(act_names) == len( + prev_saes + ), "act_names and prev_saes must have the same length" + else: + prev_saes = [None] * len(act_names) + + for act_name, prev_sae in zip(act_names, prev_saes): + self._reset_sae(act_name, prev_sae) + + self.setup() + + def run_with_saes( + self, + *model_args, + saes: Union[HookedSAE, List[HookedSAE]] = [], + reset_saes_end: bool = True, + **model_kwargs, + ) -> Union[ + None, + Float[torch.Tensor, "batch pos d_vocab"], + Loss, + Tuple[Float[torch.Tensor, "batch pos d_vocab"], Loss], + ]: + """Wrapper around HookedTransformer forward pass. + + Runs the model with the given SAEs attached for one forward pass, then removes them. By default, will reset all SAEs to original state after. + + Args: + *model_args: Positional arguments for the model forward pass + saes: (Union[HookedSAE, List[HookedSAE]]) The SAEs to be attached for this forward pass + reset_saes_end (bool): If True, all SAEs added during this run are removed at the end, and previously attached SAEs are restored to their original state. Default is True. + **model_kwargs: Keyword arguments for the model forward pass + """ + with self.saes(saes=saes, reset_saes_end=reset_saes_end): + return self(*model_args, **model_kwargs) + + def run_with_cache_with_saes( + self, + *model_args, + saes: Union[HookedSAE, List[HookedSAE]] = [], + reset_saes_end: bool = True, + return_cache_object=True, + remove_batch_dim=False, + **kwargs, + ) -> Tuple[ + Union[ + None, + Float[torch.Tensor, "batch pos d_vocab"], + Loss, + Tuple[Float[torch.Tensor, "batch pos d_vocab"], Loss], + ], + Union[ActivationCache, Dict[str, torch.Tensor]], + ]: + """Wrapper around 'run_with_cache' in HookedTransformer. + + Attaches given SAEs before running the model with cache and then removes them. + By default, will reset all SAEs to original state after. + + Args: + *model_args: Positional arguments for the model forward pass + saes: (Union[HookedSAE, List[HookedSAE]]) The SAEs to be attached for this forward pass + reset_saes_end: (bool) If True, all SAEs added during this run are removed at the end, and previously attached SAEs are restored to their original state. Default is True. + return_cache_object: (bool) if True, this will return an ActivationCache object, with a bunch of + useful HookedTransformer specific methods, otherwise it will return a dictionary of + activations as in HookedRootModule. + remove_batch_dim: (bool) Whether to remove the batch dimension (only works for batch_size==1). Defaults to False. + **kwargs: Keyword arguments for the model forward pass + """ + with self.saes(saes=saes, reset_saes_end=reset_saes_end): + return self.run_with_cache( + *model_args, + return_cache_object=return_cache_object, + remove_batch_dim=remove_batch_dim, + **kwargs, + ) + + def run_with_hooks_with_saes( + self, + *model_args, + saes: Union[HookedSAE, List[HookedSAE]] = [], + reset_saes_end: bool = True, + fwd_hooks: List[Tuple[Union[str, Callable], Callable]] = [], + bwd_hooks: List[Tuple[Union[str, Callable], Callable]] = [], + reset_hooks_end=True, + clear_contexts=False, + **model_kwargs, + ): + """Wrapper around 'run_with_hooks' in HookedTransformer. + + Attaches the given SAEs to the model before running the model with hooks and then removes them. + By default, will reset all SAEs to original state after. + + Args: + *model_args: Positional arguments for the model forward pass + act_names: (Union[HookedSAE, List[HookedSAE]]) The SAEs to be attached for this forward pass + reset_saes_end: (bool) If True, all SAEs added during this run are removed at the end, and previously attached SAEs are restored to their original state. (default: True) + fwd_hooks: (List[Tuple[Union[str, Callable], Callable]]) List of forward hooks to apply + bwd_hooks: (List[Tuple[Union[str, Callable], Callable]]) List of backward hooks to apply + reset_hooks_end: (bool) Whether to reset the hooks at the end of the forward pass (default: True) + clear_contexts: (bool) Whether to clear the contexts at the end of the forward pass (default: False) + **model_kwargs: Keyword arguments for the model forward pass + """ + with self.saes(saes=saes, reset_saes_end=reset_saes_end): + return self.run_with_hooks( + *model_args, + fwd_hooks=fwd_hooks, + bwd_hooks=bwd_hooks, + reset_hooks_end=reset_hooks_end, + clear_contexts=clear_contexts, + **model_kwargs, + ) + + @contextmanager + def saes( + self, + saes: Union[HookedSAE, List[HookedSAE]] = [], + reset_saes_end: bool = True, + ): + """ + A context manager for adding temporary SAEs to the model. + See HookedTransformer.hooks for a similar context manager for hooks. + By default will keep track of previously attached SAEs, and restore them when the context manager exits. + + Example: + + .. code-block:: python + + from transformer_lens import HookedSAETransformer, HookedSAE, HookedSAEConfig + + model = HookedSAETransformer.from_pretrained('gpt2-small') + sae_cfg = HookedSAEConfig(...) + sae = HookedSAE(sae_cfg) + with model.saes(saes=[sae]): + spliced_logits = model(text) + + + Args: + saes (Union[HookedSAE, List[HookedSAE]]): SAEs to be attached. + reset_saes_end (bool): If True, removes all SAEs added by this context manager when the context manager exits, returning previously attached SAEs to their original state. + """ + act_names_to_reset = [] + prev_saes = [] + if isinstance(saes, HookedSAE): + saes = [saes] + try: + for sae in saes: + act_names_to_reset.append(sae.cfg.hook_name) + prev_saes.append(self.acts_to_saes.get(sae.cfg.hook_name, None)) + self.add_sae(sae) + yield self + finally: + if reset_saes_end: + self.reset_saes(act_names_to_reset, prev_saes) diff --git a/transformer_lens/__init__.py b/transformer_lens/__init__.py index e2fb1484b..9ab2acea7 100644 --- a/transformer_lens/__init__.py +++ b/transformer_lens/__init__.py @@ -10,6 +10,9 @@ from .FactoredMatrix import FactoredMatrix from .ActivationCache import ActivationCache from .HookedTransformer import HookedTransformer +from .HookedSAEConfig import HookedSAEConfig +from .HookedSAE import HookedSAE +from .HookedSAETransformer import HookedSAETransformer from .SVDInterpreter import SVDInterpreter from .HookedEncoder import HookedEncoder from . import head_detector