diff --git a/.doctrees/environment.pickle b/.doctrees/environment.pickle index 6f0ac1f..cddddcb 100644 Binary files a/.doctrees/environment.pickle and b/.doctrees/environment.pickle differ diff --git a/openapi.json b/openapi.json index 4f7194b..6a8ce37 100644 --- a/openapi.json +++ b/openapi.json @@ -1 +1 @@ -{"openapi":"3.1.0","info":{"title":"FastAPI","version":"0.1.0"},"paths":{"/log":{"post":{"tags":["profile"],"summary":"Profile tabular data","description":"Profile tabular data. The Swagger UI isn't able to call this currently.\n\n## Sample curl request:\n\n```bash\ncurl -X 'POST' -H \"X-API-Key: \" -H \"Content-Type: application/json\" 'http://localhost:8000/log' --data-raw '{\n \"datasetId\": \"model-62\",\n \"multiple\": {\n \"columns\": [ \"age\", \"workclass\", \"fnlwgt\", \"education\" ],\n \"data\": [\n [ 25, \"Private\", 226802, \"11th\" ]\n ]\n }\n}'\n```\n## Sample Python client request:\n```python\nfrom whylogs_container_client import AuthenticatedClient\nimport whylogs_container_client.api.profile.log as Log\nfrom whylogs_container_client.models import LogRequest, LogMultiple\nfrom datetime import datetime\n\nclient = AuthenticatedClient(base_url=\"http://localhost:8000\", token=\"password\", prefix=\"\", auth_header_name=\"X-API-Key\")\n\ndata = LogRequest(\n dataset_id=\"model-1\",\n timestamp=int(datetime.now().timestamp() * 1000),\n multiple=LogMultiple(\n columns=[\"col1\", \"col2\"],\n data=[[1, 2], [3, 4]],\n )\n)\n\nresponse = Log.sync_detailed(client=client, json_body=data)\nif response.status_code != 200:\n raise Exception(f\"Failed to log data. Status code: {response.status_code}\")\n# API is async, it won't fail and has no return body\n```\n\n## Sample Python request (using `requests`):\n```python\nimport requests\n\n# Define your API key\napi_key = \"\"\n\n# API endpoint\nurl = 'http://localhost:8000/log'\n\n# Sample data\ndata = {\n \"datasetId\": \"model-62\",\n \"multiple\": {\n \"columns\": [\"age\", \"workclass\", \"fnlwgt\", \"education\"],\n \"data\": [\n [25, \"Private\", 226802, \"11th\"]\n ]\n }\n}\n\n# Make the POST request\nheaders = {\"X-API-Key\": api_key}\nresponse = requests.post(url, json=data, headers=headers)\n```","operationId":"log","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/LogRequest"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/log-embeddings":{"post":{"tags":["profile"],"summary":"Profile embeddings","description":"This endpoint requires a custom configuration to set up before hand. See https://docs.whylabs.ai/docs/integrations-whylogs-container/\nfor setting up embeddings support.\n\nLog embeddings data. The Swagger UI isn't able to call this currently.\n\n## Sample curl request:\n\n```bash\ncurl -X 'POST' -H \"X-API-Key: \" -H \"Content-Type: application/octet-stream\" 'http://localhost:8000/log-embeddings' --data-raw '{\n \"datasetId\": \"model-62\",\n \"timestamp\": 1634235000,\n \"embeddings\": {\n \"embeddings\": [[0.12, 0.45, 0.33, 0.92]]\n }\n}'\n```\n\n## Sample Python request (using `requests`):\n```python\nimport requests\n\n# Define your API key\napi_key = \"\"\n\n# API endpoint\nurl = 'http://localhost:8000/log-embeddings'\n\n# Sample data\ndata = {\n \"datasetId\": \"model-62\",\n \"timestamp\": 1634235000, # an example timestamp\n \"embeddings\": {\n \"embeddings\": [[0.12, 0.45, 0.33, 0.92]]\n }\n}\n\n# Make the POST request\nheaders = {\"X-API-Key\": api_key, \"Content-Type\": \"application/octet-stream\"}\nresponse = requests.post(url, json=data, headers=headers)\n```","operationId":"log_embeddings","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/LogEmbeddingRequest"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/log-pubsub":{"post":{"tags":["profile"],"summary":"Log Pubsub","operationId":"log_pubsub","responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}}}}},"/log-pubsub-embeddings":{"post":{"tags":["profile"],"summary":"Log Pubsub Embeddings","operationId":"log_pubsub_embedding","responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}}}}},"/log/llm":{"post":{"tags":["llm"],"summary":"Evaluate and log a single prompt/response pair using langkit asynchronously.","description":"This is a convenience wrapper around the llm request type for calling /log, which accepts bulk data.","operationId":"log_llm","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/LLMValidateRequest"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/evaluate/sqs":{"post":{"tags":["llm"],"summary":"Evaluate and log a single prompt/response pair asynchronously using sqs.","operationId":"evaluate_sqs","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/LLMValidateRequest"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/evaluate":{"post":{"tags":["llm"],"summary":"Evaluate and log a single prompt/response pair using langkit.","description":"Run langkit evaluation and return the validation results, as well as the generated metrics.\n\nArgs:\n log (bool, optional): Determines if logging to WhyLabs is enabled for the request. Defaults to True.","operationId":"evaluate","parameters":[{"name":"log","in":"query","required":false,"schema":{"type":"boolean","default":true,"title":"Log"}},{"name":"perf_info","in":"query","required":false,"schema":{"type":"boolean","default":false,"title":"Perf Info"}}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/LLMValidateRequest"}}}},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/EvaluationResult"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/debug/evaluate":{"post":{"tags":["debug"],"summary":"Evaluate and log a single prompt/response pair using langkit and a policy file.","operationId":"debug_evaluate","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/DebugLLMValidateRequest"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/EvaluationResult"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/list_metrics":{"get":{"tags":["llm"],"summary":"Get a list of available metrics that can be referenced in policies.","description":"Get a list of available metrics that can be referenced in policies.","operationId":"list_metrics","responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AvailableMetrics"}}}}}}},"/policy":{"get":{"tags":["llm"],"summary":"Get the JSON schema for policy files","operationId":"policy","parameters":[{"name":"schema_version","in":"query","required":false,"schema":{"type":"string","default":"0.0.1","title":"Schema Version"}}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"type":"object","title":"Response Policy"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/publish":{"post":{"tags":["manage"],"summary":"Publish Profiles","operationId":"publish","responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}}}}},"/health":{"get":{"tags":["manage"],"summary":"Health","operationId":"health","responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}}}}},"/health/llm/deep":{"get":{"tags":["manage"],"summary":"Deep Health","operationId":"health_llm_deep","responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}}}}},"/status":{"get":{"tags":["manage"],"summary":"Status","operationId":"status","responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ProcessLoggerStatusResponse"}}}}}}},"/logDebugInfo":{"post":{"tags":["manage"],"summary":"Log Debug Info","description":"Log the output of /status","operationId":"debug_info","responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}}}}}},"components":{"schemas":{"AvailableMetrics":{"properties":{"metrics_names":{"items":{"type":"string"},"type":"array","title":"Metrics Names"}},"type":"object","required":["metrics_names"],"title":"AvailableMetrics"},"BlockAction":{"properties":{"block_message":{"type":"string","title":"Block Message"},"action_type":{"const":"block","title":"Action Type","default":"block"},"is_action_block":{"type":"boolean","title":"Is Action Block"}},"type":"object","required":["block_message","is_action_block"],"title":"BlockAction"},"DebugLLMValidateRequest":{"properties":{"prompt":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Prompt"},"response":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Response"},"context":{"anyOf":[{"$ref":"#/components/schemas/InputContext"},{"type":"null"}]},"id":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Id"},"datasetId":{"type":"string","title":"Datasetid"},"timestamp":{"type":"integer","title":"Timestamp"},"additional_data":{"additionalProperties":{"anyOf":[{"type":"string"},{"type":"integer"},{"type":"number"}]},"type":"object","title":"Additional Data"},"options":{"anyOf":[{"$ref":"#/components/schemas/RunOptions"},{"type":"null"}]},"policy":{"type":"string","title":"Policy"}},"additionalProperties":false,"type":"object","required":["datasetId","policy"],"title":"DebugLLMValidateRequest"},"EvaluationResult":{"properties":{"metrics":{"items":{"type":"object"},"type":"array","title":"Metrics"},"validation_results":{"$ref":"#/components/schemas/ValidationResult"},"perf_info":{"anyOf":[{"$ref":"#/components/schemas/RunPerf"},{"type":"null"}]},"action":{"oneOf":[{"$ref":"#/components/schemas/PassAction"},{"$ref":"#/components/schemas/BlockAction"}],"title":"Action","discriminator":{"propertyName":"action_type","mapping":{"block":"#/components/schemas/BlockAction","pass":"#/components/schemas/PassAction"}}},"score_perf_info":{"anyOf":[{"$ref":"#/components/schemas/RunPerf"},{"type":"null"}]},"scores":{"items":{"type":"object"},"type":"array","title":"Scores","default":[]},"metadata":{"type":"object","title":"Metadata","default":{}}},"type":"object","required":["metrics","validation_results","perf_info","action"],"title":"EvaluationResult"},"HTTPValidationError":{"properties":{"detail":{"items":{"$ref":"#/components/schemas/ValidationError"},"type":"array","title":"Detail"}},"type":"object","title":"HTTPValidationError"},"InputContext":{"properties":{"entries":{"items":{"$ref":"#/components/schemas/InputContextItem"},"type":"array","title":"Entries"}},"type":"object","required":["entries"],"title":"InputContext"},"InputContextItem":{"properties":{"content":{"type":"string","title":"Content"},"metadata":{"additionalProperties":{"type":"string"},"type":"object","title":"Metadata"}},"type":"object","required":["content"],"title":"InputContextItem"},"LLMValidateRequest":{"properties":{"prompt":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Prompt"},"response":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Response"},"context":{"anyOf":[{"$ref":"#/components/schemas/InputContext"},{"type":"null"}]},"id":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Id"},"datasetId":{"type":"string","title":"Datasetid"},"timestamp":{"type":"integer","title":"Timestamp"},"additional_data":{"additionalProperties":{"anyOf":[{"type":"string"},{"type":"integer"},{"type":"number"}]},"type":"object","title":"Additional Data"},"options":{"anyOf":[{"$ref":"#/components/schemas/RunOptions"},{"type":"null"}]}},"additionalProperties":false,"type":"object","required":["datasetId"],"title":"LLMValidateRequest"},"LogEmbeddingRequest":{"properties":{"dataset_id":{"type":"string","title":"Dataset Id"},"timestamp":{"type":"integer","title":"Timestamp"},"embeddings":{"additionalProperties":{"anyOf":[{"items":{"items":{"type":"number"},"type":"array"},"type":"array"},{"items":{"items":{"type":"integer"},"type":"array"},"type":"array"},{"items":{"items":{"type":"string"},"type":"array"},"type":"array"}]},"type":"object","title":"Embeddings"}},"type":"object","required":["dataset_id","timestamp","embeddings"],"title":"LogEmbeddingRequest"},"LogMultiple":{"properties":{"columns":{"items":{"type":"string"},"type":"array","title":"Columns"},"data":{"items":{"items":{"anyOf":[{"type":"string"},{"type":"integer"},{"type":"number"},{"type":"boolean"},{"items":{"type":"number"},"type":"array"},{"items":{"type":"integer"},"type":"array"},{"items":{"type":"string"},"type":"array"},{"type":"null"}]},"type":"array"},"type":"array","title":"Data"}},"type":"object","required":["columns","data"],"title":"LogMultiple"},"LogRequest":{"properties":{"datasetId":{"type":"string","title":"Datasetid"},"multiple":{"$ref":"#/components/schemas/LogMultiple"},"timestamp":{"anyOf":[{"type":"integer"},{"type":"null"}],"title":"Timestamp"}},"type":"object","required":["datasetId","multiple"],"title":"LogRequest"},"LoggerStatusResponse":{"properties":{"dataset_timestamps":{"type":"integer","title":"Dataset Timestamps"},"dataset_profiles":{"type":"integer","title":"Dataset Profiles"},"segment_caches":{"type":"integer","title":"Segment Caches"},"writers":{"type":"integer","title":"Writers"},"pending_writables":{"type":"integer","title":"Pending Writables"},"pending_views":{"items":{"type":"string"},"type":"array","title":"Pending Views"},"views":{"items":{"type":"string"},"type":"array","title":"Views"}},"type":"object","required":["dataset_timestamps","dataset_profiles","segment_caches","writers","pending_writables","pending_views","views"],"title":"LoggerStatusResponse"},"MetricFilterOptions":{"properties":{"by_required_inputs":{"anyOf":[{"items":{"items":{"type":"string"},"type":"array"},"type":"array"},{"type":"null"}],"title":"By Required Inputs"}},"type":"object","title":"MetricFilterOptions"},"PassAction":{"properties":{"action_type":{"const":"pass","title":"Action Type","default":"pass"},"is_action_pass":{"type":"boolean","title":"Is Action Pass"}},"type":"object","required":["is_action_pass"],"title":"PassAction"},"ProcessLoggerStatusResponse":{"properties":{"version":{"type":"string","title":"Version"},"statuses":{"additionalProperties":{"$ref":"#/components/schemas/LoggerStatusResponse"},"type":"object","title":"Statuses"}},"type":"object","required":["version","statuses"],"title":"ProcessLoggerStatusResponse"},"RunOptions":{"properties":{"metric_filter":{"anyOf":[{"$ref":"#/components/schemas/MetricFilterOptions"},{"type":"null"}]}},"type":"object","title":"RunOptions"},"RunPerf":{"properties":{"init_total_sec":{"type":"number","title":"Init Total Sec"},"metrics_time_sec":{"additionalProperties":{"type":"number"},"type":"object","title":"Metrics Time Sec"},"metrics_total_sec":{"type":"number","title":"Metrics Total Sec"},"context_time_sec":{"additionalProperties":{"type":"number"},"type":"object","title":"Context Time Sec"},"context_total_sec":{"type":"number","title":"Context Total Sec"},"validation_total_sec":{"type":"number","title":"Validation Total Sec"},"workflow_total_sec":{"type":"number","title":"Workflow Total Sec"}},"type":"object","required":["init_total_sec","metrics_time_sec","metrics_total_sec","context_time_sec","context_total_sec","validation_total_sec","workflow_total_sec"],"title":"RunPerf"},"ValidationError":{"properties":{"loc":{"items":{"anyOf":[{"type":"string"},{"type":"integer"}]},"type":"array","title":"Location"},"msg":{"type":"string","title":"Message"},"type":{"type":"string","title":"Error Type"}},"type":"object","required":["loc","msg","type"],"title":"ValidationError"},"ValidationFailure":{"properties":{"id":{"type":"string","title":"Id"},"metric":{"type":"string","title":"Metric"},"details":{"type":"string","title":"Details"},"value":{"anyOf":[{"type":"integer"},{"type":"number"},{"type":"string"},{"type":"null"}],"title":"Value"},"upper_threshold":{"anyOf":[{"type":"number"},{"type":"null"}],"title":"Upper Threshold"},"lower_threshold":{"anyOf":[{"type":"number"},{"type":"null"}],"title":"Lower Threshold"},"allowed_values":{"anyOf":[{"items":{"anyOf":[{"type":"string"},{"type":"number"},{"type":"integer"}]},"type":"array"},{"type":"null"}],"title":"Allowed Values"},"disallowed_values":{"anyOf":[{"items":{"anyOf":[{"type":"string"},{"type":"number"},{"type":"integer"}]},"type":"array"},{"type":"null"}],"title":"Disallowed Values"},"must_be_none":{"anyOf":[{"type":"boolean"},{"type":"null"}],"title":"Must Be None"},"must_be_non_none":{"anyOf":[{"type":"boolean"},{"type":"null"}],"title":"Must Be Non None"},"failure_level":{"type":"string","enum":["flag","block"],"title":"Failure Level","default":"block"}},"type":"object","required":["id","metric","details","value"],"title":"ValidationFailure"},"ValidationResult":{"properties":{"report":{"items":{"$ref":"#/components/schemas/ValidationFailure"},"type":"array","title":"Report"}},"type":"object","title":"ValidationResult"}}}} \ No newline at end of file +{"openapi":"3.1.0","info":{"title":"FastAPI","version":"0.1.0"},"paths":{"/log":{"post":{"tags":["profile"],"summary":"Profile tabular data","description":"Profile tabular data. The Swagger UI isn't able to call this currently.\n\n## Sample curl request:\n\n```bash\ncurl -X 'POST' -H \"X-API-Key: \" -H \"Content-Type: application/json\" 'http://localhost:8000/log' --data-raw '{\n \"datasetId\": \"model-62\",\n \"multiple\": {\n \"columns\": [ \"age\", \"workclass\", \"fnlwgt\", \"education\" ],\n \"data\": [\n [ 25, \"Private\", 226802, \"11th\" ]\n ]\n }\n}'\n```\n## Sample Python client request:\n```python\nfrom whylogs_container_client import AuthenticatedClient\nimport whylogs_container_client.api.profile.log as Log\nfrom whylogs_container_client.models import LogRequest, LogMultiple\nfrom datetime import datetime\n\nclient = AuthenticatedClient(base_url=\"http://localhost:8000\", token=\"password\", prefix=\"\", auth_header_name=\"X-API-Key\")\n\ndata = LogRequest(\n dataset_id=\"model-1\",\n timestamp=int(datetime.now().timestamp() * 1000),\n multiple=LogMultiple(\n columns=[\"col1\", \"col2\"],\n data=[[1, 2], [3, 4]],\n )\n)\n\nresponse = Log.sync_detailed(client=client, json_body=data)\nif response.status_code != 200:\n raise Exception(f\"Failed to log data. Status code: {response.status_code}\")\n# API is async, it won't fail and has no return body\n```\n\n## Sample Python request (using `requests`):\n```python\nimport requests\n\n# Define your API key\napi_key = \"\"\n\n# API endpoint\nurl = 'http://localhost:8000/log'\n\n# Sample data\ndata = {\n \"datasetId\": \"model-62\",\n \"multiple\": {\n \"columns\": [\"age\", \"workclass\", \"fnlwgt\", \"education\"],\n \"data\": [\n [25, \"Private\", 226802, \"11th\"]\n ]\n }\n}\n\n# Make the POST request\nheaders = {\"X-API-Key\": api_key}\nresponse = requests.post(url, json=data, headers=headers)\n```","operationId":"log","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/LogRequest"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/log-embeddings":{"post":{"tags":["profile"],"summary":"Profile embeddings","description":"This endpoint requires a custom configuration to set up before hand. See https://docs.whylabs.ai/docs/integrations-whylogs-container/\nfor setting up embeddings support.\n\nLog embeddings data. The Swagger UI isn't able to call this currently.\n\n## Sample curl request:\n\n```bash\ncurl -X 'POST' -H \"X-API-Key: \" -H \"Content-Type: application/octet-stream\" 'http://localhost:8000/log-embeddings' --data-raw '{\n \"datasetId\": \"model-62\",\n \"timestamp\": 1634235000,\n \"embeddings\": {\n \"embeddings\": [[0.12, 0.45, 0.33, 0.92]]\n }\n}'\n```\n\n## Sample Python request (using `requests`):\n```python\nimport requests\n\n# Define your API key\napi_key = \"\"\n\n# API endpoint\nurl = 'http://localhost:8000/log-embeddings'\n\n# Sample data\ndata = {\n \"datasetId\": \"model-62\",\n \"timestamp\": 1634235000, # an example timestamp\n \"embeddings\": {\n \"embeddings\": [[0.12, 0.45, 0.33, 0.92]]\n }\n}\n\n# Make the POST request\nheaders = {\"X-API-Key\": api_key, \"Content-Type\": \"application/octet-stream\"}\nresponse = requests.post(url, json=data, headers=headers)\n```","operationId":"log_embeddings","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/LogEmbeddingRequest"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/log-pubsub":{"post":{"tags":["profile"],"summary":"Log Pubsub","operationId":"log_pubsub","responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}}}}},"/log-pubsub-embeddings":{"post":{"tags":["profile"],"summary":"Log Pubsub Embeddings","operationId":"log_pubsub_embedding","responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}}}}},"/publish":{"post":{"tags":["manage"],"summary":"Publish Profiles","operationId":"publish","responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}}}}},"/health":{"get":{"tags":["manage"],"summary":"Health","operationId":"health","responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}}}}},"/status":{"get":{"tags":["manage"],"summary":"Status","operationId":"status","responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/StatusResponse"}}}}}}},"/logDebugInfo":{"post":{"tags":["manage"],"summary":"Log Debug Info","description":"Log the output of /status","operationId":"debug_info","responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}}}}},"/log/llm":{"post":{"tags":["llm"],"summary":"Evaluate and log a single prompt/response pair using langkit asynchronously.","description":"This is a convenience wrapper around the llm request type for calling /log, which accepts bulk data.","operationId":"log_llm","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/LLMValidateRequest"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/evaluate/sqs":{"post":{"tags":["llm"],"summary":"Evaluate and log a single prompt/response pair asynchronously using sqs.","operationId":"evaluate_sqs","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/LLMValidateRequest"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/evaluate":{"post":{"tags":["llm"],"summary":"Evaluate and log a single prompt/response pair using langkit.","description":"Run langkit evaluation and return the validation results, as well as the generated metrics.\n\nArgs:\n log (bool, optional): Determines if logging to WhyLabs is enabled for the request. Defaults to True.","operationId":"evaluate","parameters":[{"name":"log","in":"query","required":false,"schema":{"type":"boolean","default":true,"title":"Log"}},{"name":"perf_info","in":"query","required":false,"schema":{"type":"boolean","default":false,"title":"Perf Info"}}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/LLMValidateRequest"}}}},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/EvaluationResult"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/debug/evaluate":{"post":{"tags":["debug"],"summary":"Evaluate and log a single prompt/response pair using langkit and a policy file.","operationId":"debug_evaluate","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/DebugLLMValidateRequest"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/EvaluationResult"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/list_metrics":{"get":{"tags":["llm"],"summary":"Get a list of available metrics that can be referenced in policies.","description":"Get a list of available metrics that can be referenced in policies.","operationId":"list_metrics","responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AvailableMetrics"}}}}}}},"/policy":{"get":{"tags":["llm"],"summary":"Get the JSON schema for policy files","operationId":"policy","parameters":[{"name":"schema_version","in":"query","required":false,"schema":{"type":"string","default":"0.0.1","title":"Schema Version"}}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"type":"object","title":"Response Policy"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/health/llm/deep":{"get":{"tags":["manage"],"summary":"Deep Health","operationId":"health_llm_deep","responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}}}}}},"components":{"schemas":{"AvailableMetrics":{"properties":{"metrics_names":{"items":{"type":"string"},"type":"array","title":"Metrics Names"}},"type":"object","required":["metrics_names"],"title":"AvailableMetrics"},"BlockAction":{"properties":{"block_message":{"type":"string","title":"Block Message"},"action_type":{"const":"block","title":"Action Type","default":"block"},"is_action_block":{"type":"boolean","title":"Is Action Block"}},"type":"object","required":["block_message","is_action_block"],"title":"BlockAction"},"DebugLLMValidateRequest":{"properties":{"prompt":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Prompt"},"response":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Response"},"context":{"anyOf":[{"$ref":"#/components/schemas/InputContext"},{"type":"null"}]},"id":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Id"},"datasetId":{"type":"string","title":"Datasetid"},"timestamp":{"type":"integer","title":"Timestamp"},"additional_data":{"additionalProperties":{"anyOf":[{"type":"string"},{"type":"integer"},{"type":"number"}]},"type":"object","title":"Additional Data"},"options":{"anyOf":[{"$ref":"#/components/schemas/RunOptions"},{"type":"null"}]},"policy":{"type":"string","title":"Policy"}},"additionalProperties":false,"type":"object","required":["datasetId","policy"],"title":"DebugLLMValidateRequest"},"EvaluationResult":{"properties":{"metrics":{"items":{"type":"object"},"type":"array","title":"Metrics"},"validation_results":{"$ref":"#/components/schemas/ValidationResult"},"perf_info":{"anyOf":[{"$ref":"#/components/schemas/RunPerf"},{"type":"null"}]},"action":{"oneOf":[{"$ref":"#/components/schemas/PassAction"},{"$ref":"#/components/schemas/BlockAction"}],"title":"Action","discriminator":{"propertyName":"action_type","mapping":{"block":"#/components/schemas/BlockAction","pass":"#/components/schemas/PassAction"}}},"score_perf_info":{"anyOf":[{"$ref":"#/components/schemas/RunPerf"},{"type":"null"}]},"scores":{"items":{"type":"object"},"type":"array","title":"Scores","default":[]},"metadata":{"type":"object","title":"Metadata","default":{}}},"type":"object","required":["metrics","validation_results","perf_info","action"],"title":"EvaluationResult"},"HTTPValidationError":{"properties":{"detail":{"items":{"$ref":"#/components/schemas/ValidationError"},"type":"array","title":"Detail"}},"type":"object","title":"HTTPValidationError"},"InputContext":{"properties":{"entries":{"items":{"$ref":"#/components/schemas/InputContextItem"},"type":"array","title":"Entries"}},"type":"object","required":["entries"],"title":"InputContext"},"InputContextItem":{"properties":{"content":{"type":"string","title":"Content"},"metadata":{"additionalProperties":{"type":"string"},"type":"object","title":"Metadata"}},"type":"object","required":["content"],"title":"InputContextItem"},"LLMValidateRequest":{"properties":{"prompt":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Prompt"},"response":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Response"},"context":{"anyOf":[{"$ref":"#/components/schemas/InputContext"},{"type":"null"}]},"id":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Id"},"datasetId":{"type":"string","title":"Datasetid"},"timestamp":{"type":"integer","title":"Timestamp"},"additional_data":{"additionalProperties":{"anyOf":[{"type":"string"},{"type":"integer"},{"type":"number"}]},"type":"object","title":"Additional Data"},"options":{"anyOf":[{"$ref":"#/components/schemas/RunOptions"},{"type":"null"}]}},"additionalProperties":false,"type":"object","required":["datasetId"],"title":"LLMValidateRequest"},"LogEmbeddingRequest":{"properties":{"dataset_id":{"type":"string","title":"Dataset Id"},"timestamp":{"type":"integer","title":"Timestamp"},"embeddings":{"additionalProperties":{"anyOf":[{"items":{"items":{"type":"number"},"type":"array"},"type":"array"},{"items":{"items":{"type":"integer"},"type":"array"},"type":"array"},{"items":{"items":{"type":"string"},"type":"array"},"type":"array"}]},"type":"object","title":"Embeddings"}},"type":"object","required":["dataset_id","timestamp","embeddings"],"title":"LogEmbeddingRequest"},"LogMultiple":{"properties":{"columns":{"items":{"type":"string"},"type":"array","title":"Columns"},"data":{"items":{"items":{"anyOf":[{"type":"string"},{"type":"integer"},{"type":"number"},{"type":"boolean"},{"items":{"type":"number"},"type":"array"},{"items":{"type":"integer"},"type":"array"},{"items":{"type":"string"},"type":"array"},{"type":"null"}]},"type":"array"},"type":"array","title":"Data"}},"type":"object","required":["columns","data"],"title":"LogMultiple"},"LogRequest":{"properties":{"datasetId":{"type":"string","title":"Datasetid"},"multiple":{"$ref":"#/components/schemas/LogMultiple"},"timestamp":{"anyOf":[{"type":"integer"},{"type":"null"}],"title":"Timestamp"}},"type":"object","required":["datasetId","multiple"],"title":"LogRequest"},"LoggerStatusResponse":{"properties":{"dataset_timestamps":{"type":"integer","title":"Dataset Timestamps"},"dataset_profiles":{"type":"integer","title":"Dataset Profiles"},"segment_caches":{"type":"integer","title":"Segment Caches"},"writers":{"type":"integer","title":"Writers"},"pending_writables":{"type":"integer","title":"Pending Writables"},"pending_views":{"items":{"type":"string"},"type":"array","title":"Pending Views"},"views":{"items":{"type":"string"},"type":"array","title":"Views"}},"type":"object","required":["dataset_timestamps","dataset_profiles","segment_caches","writers","pending_writables","pending_views","views"],"title":"LoggerStatusResponse"},"MetricFilterOptions":{"properties":{"by_required_inputs":{"anyOf":[{"items":{"items":{"type":"string"},"type":"array"},"type":"array"},{"type":"null"}],"title":"By Required Inputs"}},"type":"object","title":"MetricFilterOptions"},"PassAction":{"properties":{"action_type":{"const":"pass","title":"Action Type","default":"pass"},"is_action_pass":{"type":"boolean","title":"Is Action Pass"}},"type":"object","required":["is_action_pass"],"title":"PassAction"},"RunOptions":{"properties":{"metric_filter":{"anyOf":[{"$ref":"#/components/schemas/MetricFilterOptions"},{"type":"null"}]}},"type":"object","title":"RunOptions"},"RunPerf":{"properties":{"init_total_sec":{"type":"number","title":"Init Total Sec"},"metrics_time_sec":{"additionalProperties":{"type":"number"},"type":"object","title":"Metrics Time Sec"},"metrics_total_sec":{"type":"number","title":"Metrics Total Sec"},"context_time_sec":{"additionalProperties":{"type":"number"},"type":"object","title":"Context Time Sec"},"context_total_sec":{"type":"number","title":"Context Total Sec"},"validation_total_sec":{"type":"number","title":"Validation Total Sec"},"workflow_total_sec":{"type":"number","title":"Workflow Total Sec"}},"type":"object","required":["init_total_sec","metrics_time_sec","metrics_total_sec","context_time_sec","context_total_sec","validation_total_sec","workflow_total_sec"],"title":"RunPerf"},"StatusResponse":{"properties":{"version":{"type":"string","title":"Version"},"whylogs_logger_status":{"additionalProperties":{"$ref":"#/components/schemas/LoggerStatusResponse"},"type":"object","title":"Whylogs Logger Status"},"config":{"type":"object","title":"Config"}},"type":"object","required":["version","whylogs_logger_status","config"],"title":"StatusResponse"},"ValidationError":{"properties":{"loc":{"items":{"anyOf":[{"type":"string"},{"type":"integer"}]},"type":"array","title":"Location"},"msg":{"type":"string","title":"Message"},"type":{"type":"string","title":"Error Type"}},"type":"object","required":["loc","msg","type"],"title":"ValidationError"},"ValidationFailure":{"properties":{"id":{"type":"string","title":"Id"},"metric":{"type":"string","title":"Metric"},"details":{"type":"string","title":"Details"},"value":{"anyOf":[{"type":"integer"},{"type":"number"},{"type":"string"},{"type":"null"}],"title":"Value"},"upper_threshold":{"anyOf":[{"type":"number"},{"type":"null"}],"title":"Upper Threshold"},"lower_threshold":{"anyOf":[{"type":"number"},{"type":"null"}],"title":"Lower Threshold"},"allowed_values":{"anyOf":[{"items":{"anyOf":[{"type":"string"},{"type":"number"},{"type":"integer"}]},"type":"array"},{"type":"null"}],"title":"Allowed Values"},"disallowed_values":{"anyOf":[{"items":{"anyOf":[{"type":"string"},{"type":"number"},{"type":"integer"}]},"type":"array"},{"type":"null"}],"title":"Disallowed Values"},"must_be_none":{"anyOf":[{"type":"boolean"},{"type":"null"}],"title":"Must Be None"},"must_be_non_none":{"anyOf":[{"type":"boolean"},{"type":"null"}],"title":"Must Be Non None"},"failure_level":{"type":"string","enum":["flag","block"],"title":"Failure Level","default":"block"}},"type":"object","required":["id","metric","details","value"],"title":"ValidationFailure"},"ValidationResult":{"properties":{"report":{"items":{"$ref":"#/components/schemas/ValidationFailure"},"type":"array","title":"Report"}},"type":"object","title":"ValidationResult"}}}} \ No newline at end of file diff --git a/whylogs-container-python.html b/whylogs-container-python.html index f97f759..328643c 100644 --- a/whylogs-container-python.html +++ b/whylogs-container-python.html @@ -504,7 +504,7 @@ -
Documentation Powered by ReDoc
Documentation Powered by ReDoc
Sample Python request (using

Request samples

Content type
application/json
{
  • "dataset_id": "string",
  • "timestamp": 0,
  • "embeddings": {
    }
}

Response samples

Content type
application/json
null

Log Pubsub

Responses

Response samples

Content type
application/json
null

Log Pubsub Embeddings

Responses

Response samples

Content type
application/json
null

llm

Evaluate and log a single prompt/response pair using langkit asynchronously.

This is a convenience wrapper around the llm request type for calling /log, which accepts bulk data.

+

Response samples

Content type
application/json
null

manage

Publish Profiles

Responses

Response samples

Content type
application/json
null

Health

Responses

Response samples

Content type
application/json
null

Status

Responses

Response samples

Content type
application/json
{
  • "version": "string",
  • "whylogs_logger_status": {
    },
  • "config": { }
}

Log Debug Info

Log the output of /status

+

Responses

Response samples

Content type
application/json
null

Deep Health

Responses

Response samples

Content type
application/json
null

llm

Evaluate and log a single prompt/response pair using langkit asynchronously.

This is a convenience wrapper around the llm request type for calling /log, which accepts bulk data.

Request Body schema: application/json
string or null (Prompt)
string or null (Response)
InputContext (object) or null
string or null (Id)
datasetId
required
string (Datasetid)
timestamp
integer (Timestamp)
object (Additional Data)
RunOptions (object) or null

Responses

Request samples

Content type
application/json
{
  • "prompt": "string",
  • "response": "string",
  • "context": {
    },
  • "id": "string",
  • "datasetId": "string",
  • "timestamp": 0,
  • "additional_data": {
    },
  • "options": {
    }
}

Response samples

Content type
application/json
null

Evaluate and log a single prompt/response pair asynchronously using sqs.

Request Body schema: application/json
string or null (Prompt)
string or null (Response)
InputContext (object) or null
string or null (Id)
datasetId
required
string (Datasetid)
timestamp
integer (Timestamp)
object (Additional Data)
RunOptions (object) or null

Responses

Request samples

Content type
application/json
{
  • "prompt": "string",
  • "response": "string",
  • "context": {
    },
  • "id": "string",
  • "datasetId": "string",
  • "timestamp": 0,
  • "additional_data": {
    },
  • "options": {
    }
}

Response samples

Content type
application/json
null

Evaluate and log a single prompt/response pair asynchronously using sqs.

Request Body schema: application/json
string or null (Prompt)
string or null (Response)
InputContext (object) or null
string or null (Id)
datasetId
required
string (Datasetid)
timestamp
integer (Timestamp)
object (Additional Data)
RunOptions (object) or null

Responses

Request samples

Content type
application/json
{
  • "prompt": "string",
  • "response": "string",
  • "context": {
    },
  • "id": "string",
  • "datasetId": "string",
  • "timestamp": 0,
  • "additional_data": {
    },
  • "options": {
    }
}

Response samples

Content type
application/json
null

Evaluate and log a single prompt/response pair using langkit.

Run langkit evaluation and return the validation results, as well as the generated metrics.

+

Request samples

Content type
application/json
{
  • "prompt": "string",
  • "response": "string",
  • "context": {
    },
  • "id": "string",
  • "datasetId": "string",
  • "timestamp": 0,
  • "additional_data": {
    },
  • "options": {
    }
}

Response samples

Content type
application/json
null

Evaluate and log a single prompt/response pair using langkit.

Run langkit evaluation and return the validation results, as well as the generated metrics.

Args: log (bool, optional): Determines if logging to WhyLabs is enabled for the request. Defaults to True.

query Parameters
log
boolean (Log)
Default: true
perf_info
boolean (Perf Info)
Default: false
Request Body schema: application/json
string or null (Prompt)
string or null (Response)
InputContext (object) or null
string or null (Id)
datasetId
required
string (Datasetid)
timestamp
integer (Timestamp)
object (Additional Data)
RunOptions (object) or null

Responses

Request samples

Content type
application/json
{
  • "prompt": "string",
  • "response": "string",
  • "context": {
    },
  • "id": "string",
  • "datasetId": "string",
  • "timestamp": 0,
  • "additional_data": {
    },
  • "options": {
    }
}

Response samples

Content type
application/json
{
  • "metrics": [
    ],
  • "validation_results": {
    },
  • "perf_info": {
    },
  • "action": {
    },
  • "score_perf_info": {
    },
  • "scores": [ ],
  • "metadata": { }
}

Get a list of available metrics that can be referenced in policies.

Get a list of available metrics that can be referenced in policies.

+

Request samples

Content type
application/json
{
  • "prompt": "string",
  • "response": "string",
  • "context": {
    },
  • "id": "string",
  • "datasetId": "string",
  • "timestamp": 0,
  • "additional_data": {
    },
  • "options": {
    }
}

Response samples

Content type
application/json
{
  • "metrics": [
    ],
  • "validation_results": {
    },
  • "perf_info": {
    },
  • "action": {
    },
  • "score_perf_info": {
    },
  • "scores": [ ],
  • "metadata": { }
}

Get a list of available metrics that can be referenced in policies.

Get a list of available metrics that can be referenced in policies.

Responses

Response samples

Content type
application/json
{
  • "metrics_names": [
    ]
}

Get the JSON schema for policy files

query Parameters
schema_version
string (Schema Version)
Default: "0.0.1"

Responses

Response samples

Content type
application/json
{
  • "metrics_names": [
    ]
}

Get the JSON schema for policy files

query Parameters
schema_version
string (Schema Version)
Default: "0.0.1"

Responses

Response samples

Content type
application/json
{ }

debug

Evaluate and log a single prompt/response pair using langkit and a policy file.

Request Body schema: application/json
string or null (Prompt)
string or null (Response)
InputContext (object) or null
string or null (Id)
datasetId
required
string (Datasetid)
timestamp
integer (Timestamp)
object (Additional Data)
RunOptions (object) or null
policy
required
string (Policy)

Responses

Response samples

Content type
application/json
{ }

debug

Evaluate and log a single prompt/response pair using langkit and a policy file.

Request Body schema: application/json
string or null (Prompt)
string or null (Response)
InputContext (object) or null
string or null (Id)
datasetId
required
string (Datasetid)
timestamp
integer (Timestamp)
object (Additional Data)
RunOptions (object) or null
policy
required
string (Policy)

Responses

Request samples

Content type
application/json
{
  • "prompt": "string",
  • "response": "string",
  • "context": {
    },
  • "id": "string",
  • "datasetId": "string",
  • "timestamp": 0,
  • "additional_data": {
    },
  • "options": {
    },
  • "policy": "string"
}

Response samples

Content type
application/json
{
  • "metrics": [
    ],
  • "validation_results": {
    },
  • "perf_info": {
    },
  • "action": {
    },
  • "score_perf_info": {
    },
  • "scores": [ ],
  • "metadata": { }
}

manage

Publish Profiles

Responses

Response samples

Content type
application/json
null

Health

Responses

Response samples

Content type
application/json
null

Deep Health

Responses

Response samples

Content type
application/json
null

Status

Responses

Response samples

Content type
application/json
{
  • "version": "string",
  • "statuses": {
    }
}

Log Debug Info

Log the output of /status

-

Responses

Response samples

Content type
application/json
null
+

Request samples

Content type
application/json
{
  • "prompt": "string",
  • "response": "string",
  • "context": {
    },
  • "id": "string",
  • "datasetId": "string",
  • "timestamp": 0,
  • "additional_data": {
    },
  • "options": {
    },
  • "policy": "string"
}

Response samples

Content type
application/json
{
  • "metrics": [
    ],
  • "validation_results": {
    },
  • "perf_info": {
    },
  • "action": {
    },
  • "score_perf_info": {
    },
  • "scores": [ ],
  • "metadata": { }
}