From 9d08d9e4bf2db4c96ee6f41b7a5aef0849d43687 Mon Sep 17 00:00:00 2001 From: John Smyth Date: Mon, 7 Oct 2024 14:20:09 -0500 Subject: [PATCH] add back docs for query snapshots --- docs/managing/workspaces.md | 50 +----- docs/query/snapshots.md | 139 +++++++++++++++ docs/reference/cli/query.md | 151 +++++------------ docs/reference/config-files/workspace.md | 158 +----------------- docs/reference/env-vars/overview.md | 12 +- .../env-vars/steampipe_snapshot_location.md | 25 +++ docs/sidebar.json | 2 + 7 files changed, 222 insertions(+), 315 deletions(-) create mode 100644 docs/query/snapshots.md create mode 100644 docs/reference/env-vars/steampipe_snapshot_location.md diff --git a/docs/managing/workspaces.md b/docs/managing/workspaces.md index 1f532f3..85dfaf4 100644 --- a/docs/managing/workspaces.md +++ b/docs/managing/workspaces.md @@ -18,6 +18,9 @@ workspace "local" { workspace "acme_prod" { workspace_database = "acme/prod" + snapshot_location = "acme/prod" + query_timeout = 600 + } ``` @@ -70,8 +73,6 @@ workspace "acme_prod" { } ``` - - - - You can even set the `install_dir` for a workspace if you want to use the data layer from another [Steampipe installation directory](https://steampipe.io/docs/reference/env-vars/steampipe_install_dir). This allows you to define workspaces that use a database from another installation directory: @@ -151,7 +131,6 @@ steampipe dashboard --workspace steampipe_2 ``` - ## Using Workspaces Workspaces may be defined in any `.spc` file in the `~/.steampipe/config` directory, but by convention they should be placed in the `~/.steampipe/config/workspaces.spc` file. @@ -217,14 +196,12 @@ steampipe query --workspace=acme_prod "select * from aws_account" ``` ## Implicit Workspaces - Named workspaces follow normal standards for HCL identifiers, thus they cannot contain the slash (`/`) character. If you pass a value to `--workspace` or `STEAMPIPE_WORKSPACE` in the form of `{identity_handle}/{workspace_handle}`, it will be interpreted as an **implicit workspace**. Implicit workspaces, as the name suggests, do not need to be specified in the `workspaces.spc` file. Instead they will be assumed -to refer to a Turbot Pipes workspace, which will be used as both the database (`workspace_database`) -and snapshot location (`snapshot_location`). +to refer to a Turbot Pipes workspace, which will be used as both the database (`workspace_database`) and snapshot location (`snapshot_location`). Essentially, `--workspace acme/dev` is equivalent to: ```hcl @@ -232,19 +209,4 @@ workspace "acme/dev" { workspace_database = "acme/dev" snapshot_location = "acme/dev" } -``` ---> - -Named workspaces follow normal standards for HCL identifiers, thus they cannot contain -the slash (`/`) character. If you pass a value to `--workspace` or `STEAMPIPE_WORKSPACE` -in the form of `{identity_handle}/{workspace_handle}`, it will be interpreted as -an **implicit workspace**. Implicit workspaces, as the name suggests, do not -need to be specified in the `workspaces.spc` file. Instead they will be assumed -to refer to a Turbot Pipes workspace, which will be used as the database (`workspace_database`). - -Essentially, `--workspace acme/dev` is equivalent to: -```hcl -workspace "acme/dev" { - workspace_database = "acme/dev" -} -``` +``` \ No newline at end of file diff --git a/docs/query/snapshots.md b/docs/query/snapshots.md new file mode 100644 index 0000000..19dc26f --- /dev/null +++ b/docs/query/snapshots.md @@ -0,0 +1,139 @@ +--- +title: Snapshots +sidebar_label: Snapshots +--- + +# Snapshots + +Steampipe allows you to take **snapshots**. A snapshot is a saved view of your query results that you can view as a [dashboard in Powerpipe](https://powerpipe.io/docs/run/dashboard) All data and metadata for a snapshot is contained in a JSON file which can be saved and viewed locally in the Powerpipe dashboard or uploaded to [Turbot Pipes](https://turbot.com/pipes/docs). Snapshots in Turbot Pipes may be shared with other Turbot Pipes users or made public (shared with anyone that has the link). + +You can create Turbot Pipes snapshots directly from the Steampipe CLI, however if you wish to subsequently [modify](https://turbot.com/pipes/docs/dashboards#managing-snapshots) them (add/remove tags, change visibility) or delete them, you must do so from the Turbot Pipes console. You may [browse the snapshot list](https://turbot.com/pipes/docs/dashboards#browsing-snapshots) in Turbot Pipes by clicking the **Snapshots** button on the top of your workspace's **Dashboards** page. + + +## Taking Snapshots + +> To upload snapshots to Turbot Pipes, you must either [log in via the `steampipe login` command](/docs/reference/cli/login) or create an [API token](https://turbot.com/pipes/docs/profile#tokens) and pass it via the [`--pipes-token`](/docs/reference/cli/overview#global-flags) flag or [`PIPES_TOKEN`](/docs/reference/env-vars/pipes_token) environment variable. + +To take a snapshot and save it to [Turbot Pipes](https://turbot.com/pipes/docs), simply add the `--snapshot` flag to your command. + +```bash +steampipe query --snapshot "select * from aws_ec2_instance" +``` + +The `--snapshot` flag will create a snapshot with `workspace` visibility in your user workspace. A snapshot with `workspace` visibility is visible only to users that have access to the workspace in which the snapshot resides -- A user must be authenticated to Turbot Pipes with permissions on the workspace. + +If you want to create a snapshot that can be shared with *anyone*, use the `--share` flag instead. This will create the snapshot with `anyone_with_link` visibility: + +```bash +steampipe query --share "select * from aws_ec2_instance" +``` + + +You can set a snapshot title in Turbot Pipes with the `--snapshot-title` argument. + +```bash +steampipe query --share --snapshot-title "Public Buckets" "select name from aws_s3_bucket where bucket_policy_is_public" +``` + + +If you wish to save to the snapshot to a different workspace, such as an org workspace, you can use the `--snapshot-location` argument with `--share` or `--snapshot`: + +```bash +steampipe query --share --snapshot-location vandelay-industries/latex "select * from aws_ec2_instance" +``` + +Note that the previous command ran the query against the *local* database, but saved the snapshot to the `vandelay-industries/latex` workspace. If you want to run the query against the remote `vandelay-industries/latex` database AND store the snapshot there, you can also add the `--database-location` argument: + +```bash +steampipe query --share --snapshot-location vandelay-industries/latex \ + --workspace-database vandelay-industries/latex \ + "select * from aws_ec2_instance" +``` + +Steampipe provides a shortcut for this though. The `--workspace` flag supports [passing the cloud workspace](/docs/managing/workspaces#implicit-workspaces): +```bash + +steampipe query --snapshot --workspace vandelay-industries/latex "select * from aws_ec2_instance" + +``` + +While not a common case, you can even run a query against a Turbot Pipes workspace database, but store the snapshot in an entirely different Turbot Pipes workspace: +```bash + +steampipe query --share --snapshot-location vandelay-industries/latex \ + --workspace vandelay-industries/latex-prod \ + "select * from aws_ec2_instance" +``` + + +## Tagging Snapshots + +You may want to tag your snapshots to make it easier to organize them. You can use the `--snapshot-tag` argument to add a tag: + +```bash +steampipe query --snapshot-tag env=local --snapshot \ + "select * from aws_ec2_instance" +``` + +Simply repeat the flag to add more than one tag: +```bash +steampipe query --snapshot-tag env=local --snapshot --snapshot-tag owner=george \ + "select * from aws_ec2_instance" + +``` + + +## Saving Snapshots to Local Files + +Turbot Pipes makes it easy to save and share your snapshots, however it is not strictly required; You can save and view snapshots using only the CLI. + +You can specify a local path in the `--snapshot-location` argument or `STEAMPIPE_SNAPSHOT_LOCATION` environment variable to save your snapshots to a directory in your filesystem: + +```bash +steampipe query --snapshot --snapshot-location . "select * from aws_account" +``` + +You can also set `snapshot_location` in a [workspace](/docs/managing/workspaces) if you wish to make it the default location. + + +Alternatively, you can use the `--export` argument to export a query, dashboard, or benchmark in the Steampipe snapshot format. This will create a file with a `.sps` extension in the current directory: + +```bash +steampipe query --export sps "select * from aws_account" +``` + +The `snapshot` export/output type is an alias for `sps`: + +```bash +steampipe query --export snapshot "select * from aws_account" +``` + +To give the file a name, simply use `{filename}.sps`, for example: + +```bash +steampipe query --export aws_accounts.sps "select * from aws_account" +``` + +Alternatively, you can write the steampipe snapshot to stdout with `--output sps` +```bash +steampipe query --output sps "select * from aws_account" > aws_accounts.sps +``` + +or `--output snapshot` +```bash +steampipe query --output snapshot "select * from aws_account" > aws_accounts.sps +``` + + +## Controlling Output +When using `--share` or `--snapshot`, the output will include the URL to view the snapshot that you created in addition to the usual output: +```bash +Snapshot uploaded to https://pipes.turbot.com/user/costanza/workspace/vandelay/snapshot/snap_abcdefghij0123456789_asdfghjklqwertyuiopzxcvbn +``` + +You can use the `--progress=false` argument to suppress displaying the URL and other progress data. This may be desirable when you are using an alternate output format, especially when piping the output to another command: + +```bash +steampipe query --snapshot --output json \ + --progress=false "select * from aws_account" | jq +``` diff --git a/docs/reference/cli/query.md b/docs/reference/cli/query.md index 9be2922..b8fbe06 100644 --- a/docs/reference/cli/query.md +++ b/docs/reference/cli/query.md @@ -24,56 +24,6 @@ steampipe query {query} [flags] ``` - ## Flags -| Flag | Description -|-|- - - - - - + + + - - + + - + @@ -149,10 +58,8 @@ steampipe query query.s3_bucket_logging_enabled - - @@ -162,14 +69,12 @@ steampipe query query.s3_bucket_logging_enabled - - @@ -200,25 +105,49 @@ steampipe query query.s3_bucket_logging_enabled - + - + + + + + + + + + + + + + + + + + + + + + + + + + + - - + +
Argument Description
Argument Description
--cloud-host Sets the Turbot Pipes host used when connecting to Turbot Pipes workspaces. DEPRECATED - Use --pipes-host. --cloud-host Sets the Turbot Pipes host used when connecting to Turbot Pipes workspaces. DEPRECATED - Use --pipes-host.
--cloud-token Sets the Turbot Pipes authentication token used when connecting to Turbot Pipes workspaces. DEPRECATED - Use --pipes-token.
--export string Export query output to a file. You may export multiple output formats by entering multiple --export arguments. If a file path is specified as an argument, its type will be inferred by the suffix. Supported export formats are sps (snapshot).
--header string Specify whether to include column headers in csv and table output (default true). --help Help for steampipe query.
--output string Select the console output format. Possible values are line, csv, json, table, snapshot (default table) .
--pipes-host Sets the Turbot Pipes host used when connecting to Turbot Pipes workspaces. See PIPES_HOST for details. Set a comma-separated list of connections to use as a prefix to the current search path for the query session.
--separator string A single character to use as a separator string for csv output (defaults to ",")
--share Create snapshot in Turbot Pipes with anyone_with_link visibility.
--snapshot Create snapshot in Turbot Pipes with the default (workspace) visibility.
--snapshot-location string The location to write snapshots - either a local file path or a Turbot Pipes workspace
--snapshot-tag string=string Specify tags to set on the snapshot. Multiple --snapshot-tag arguments may be passed.
--snapshot-title string=string The title to give a snapshot when uploading to Turbot Pipes.
--timing=string Enable or disable query execution timing: off (default), on, or verbose
--workspace-database
--workspace-database Sets the database that Steampipe will connect to. This can be local (the default) or a remote Turbot Pipes database. See STEAMPIPE_WORKSPACE_DATABASE for details.
- ## Examples Open an interactive query console: @@ -231,6 +160,16 @@ Run a specific query directly: steampipe query "select * from aws_s3_bucket" ``` +Run a query and save a [snapshot](/docs/snapshots/batch-snapshots): +```bash +steampipe query --snapshot "select * from aws_s3_bucket" +``` + +Run a query and share a [snapshot](/docs/snapshots/batch-snapshots): +```bash +steampipe query --share "select * from aws_s3_bucket" +``` + Run the SQL command in the `my_queries/my_query.sql` file: ```bash steampipe query my_queries/my_query.sql diff --git a/docs/reference/config-files/workspace.md b/docs/reference/config-files/workspace.md index b573a00..ecb4f84 100644 --- a/docs/reference/config-files/workspace.md +++ b/docs/reference/config-files/workspace.md @@ -54,24 +54,8 @@ Any unset arguments will assume the default values. | `query_timeout` | `240` for controls, unlimited otherwise | The maximum time (in seconds) a query is allowed to run before it times out.

Env: [STEAMPIPE_QUERY_TIMEOUT](/docs/reference/env-vars/steampipe_query_timeout)
CLI: `--query_timeout` | `search_path` | `public`, then alphabetical | A comma-separated list of connections to use as a custom search path for the control run. See also: [Using search_path to target connections and aggregators](https://steampipe.io/docs/guides/search-path).

CLI: `--search-path` | `search_path_prefix`| | A comma-separated list of connections to use as a prefix to the current search path for the control run. See also: [Using search_path to target connections and aggregators](https://steampipe.io/docs/guides/search-path).

CLI: `--search-path-prefix` -| `workspace_database`| `local` | Workspace database. This can be local or a remote Turbot Pipes database.

Env: [STEAMPIPE_WORKSPACE_DATABASE](/docs/reference/env-vars/steampipe_workspace_database)
CLI: `--workspace-database` - - - - +| `workspace_database`| `local` | Workspace database. This can be local or a remote Turbot Pipes database.

Env: [STEAMPIPE_WORKSPACE_DATABASE](/docs/reference/env-vars/steampipe_workspace_database)
CLI: `--workspace-database` @@ -124,156 +108,26 @@ These options often correspond to CLI flags. - - -## Examples - - -```hcl - -workspace "default" { - query_timeout = 300 -} - - workspace "all_options" { pipes_host = "pipes.turbot.com" pipes_token = "tpt_999faketoken99999999_111faketoken1111111111111" install_dir = "~/steampipe2" query_timeout = 300 workspace_database = "local" + snapshot_location = "acme/dev" search_path = "aws,aws_1,aws_2,gcp,gcp_1,gcp_2,slack,github" search_path_prefix = "aws_all" progress = true cache = true cache_ttl = 300 - options "query" { autocomplete = true header = true # true, false @@ -282,11 +136,5 @@ workspace "all_options" { separator = "," # any single char timing = "on" # on, off, verbose } - } - -``` - - - - +``` \ No newline at end of file diff --git a/docs/reference/env-vars/overview.md b/docs/reference/env-vars/overview.md index cdabcd0..4844f9d 100644 --- a/docs/reference/env-vars/overview.md +++ b/docs/reference/env-vars/overview.md @@ -35,16 +35,8 @@ Note that plugins may also support environment variables, but these are plugin-s | [STEAMPIPE_OTEL_LEVEL](reference/env-vars/steampipe_otel_level) | `NONE` | Specify which [OpenTelemetry](https://opentelemetry.io/) data to send via OTLP. | [STEAMPIPE_PLUGIN_MEMORY_MAX_MB](reference/env-vars/steampipe_plugin_memory_max_mb)| `1024` | Set a default memory soft limit for each plugin process. | [STEAMPIPE_QUERY_TIMEOUT](reference/env-vars/steampipe_query_timeout) | `240` for controls, unlimited in all other cases. | Set the amount of time to wait for a query to complete before timing out, in seconds. +| [STEAMPIPE_SNAPSHOT_LOCATION](/docs/reference/env-vars/steampipe_snapshot_location) | The Turbot Pipes user's personal workspace | Set the Turbot Pipes workspace or filesystem path for writing snapshots. | [STEAMPIPE_TELEMETRY](reference/env-vars/steampipe_telemetry) | `info` | Set the level of telemetry data to collect and send. | [STEAMPIPE_UPDATE_CHECK](reference/env-vars/steampipe_update_check)| `true` | Enable/disable automatic update checking. | [STEAMPIPE_WORKSPACE](reference/env-vars/steampipe_workspace) | `default` | Set the Steampipe workspace . This can be named workspace from `workspaces.spc` or a remote Turbot Pipes workspace -| [STEAMPIPE_WORKSPACE_DATABASE](reference/env-vars/steampipe_workspace_database) | `local` | Workspace database. This can be `local` or a remote Turbot Pipes database. - - - \ No newline at end of file +| [STEAMPIPE_WORKSPACE_DATABASE](reference/env-vars/steampipe_workspace_database) | `local` | Workspace database. This can be `local` or a remote Turbot Pipes database. \ No newline at end of file diff --git a/docs/reference/env-vars/steampipe_snapshot_location.md b/docs/reference/env-vars/steampipe_snapshot_location.md new file mode 100644 index 0000000..d0af11e --- /dev/null +++ b/docs/reference/env-vars/steampipe_snapshot_location.md @@ -0,0 +1,25 @@ +--- +title: STEAMPIPE_SNAPSHOT_LOCATION +sidebar_label: STEAMPIPE_SNAPSHOT_LOCATION +--- + + +# STEAMPIPE_SNAPSHOT_LOCATION + +Sets the location to write [snapshots](/docs/query/snapshots) - either a local file path or a [Turbot Pipes workspace](https://turbot.com/pipes/docs/workspaces). + +By default, Steampipe will write snapshots to your default Turbot Pipes user workspace. + +## Usage +Set the snapshot location to a local filesystem path: + +```bash +export STEAMPIPE_SNAPSHOT_LOCATION=~/my-snaps +``` + + +Set the snapshot location to a Turbot Pipes workspace: + +```bash +export STEAMPIPE_SNAPSHOT_LOCATION=vandelay-industries/latex +``` \ No newline at end of file diff --git a/docs/sidebar.json b/docs/sidebar.json index 8d81d51..adb703b 100644 --- a/docs/sidebar.json +++ b/docs/sidebar.json @@ -26,6 +26,7 @@ "items": [ "query/query-shell", "query/batch-query", + "query/snapshots", "query/third-party" ] }, @@ -179,6 +180,7 @@ "reference/env-vars/steampipe_otel_level", "reference/env-vars/steampipe_plugin_memory_max_mb", "reference/env-vars/steampipe_query_timeout", + "reference/env-vars/steampipe_snapshot_location", "reference/env-vars/steampipe_telemetry", "reference/env-vars/steampipe_update_check", "reference/env-vars/steampipe_workspace",