From f61a11f57a9ffb0efb2d6c2300516cb953d06883 Mon Sep 17 00:00:00 2001 From: Eda Z <36808054+edamamez@users.noreply.github.com> Date: Mon, 16 Dec 2024 11:13:24 -0800 Subject: [PATCH] Update to 3.2.3 (#77) Co-authored-by: github-actions[bot] --- lamini/__init__.py | 5 +- lamini/api/lamini.py | 70 +++++++++++++++++++++++++ lamini/api/model_downloader.py | 11 ++-- lamini/api/rest_requests.py | 78 ++++++++++++++++++++++------ lamini/api/utils/completion.py | 3 ++ lamini/api/utils/sql_completion.py | 40 ++++++++++++++ lamini/api/utils/sql_token_cache.py | 47 +++++++++++++++++ lamini/api/utils/supported_models.py | 6 ++- lamini/classify/lamini_classifier.py | 65 ++++++++++++++++++----- lamini/error/error.py | 14 ++++- lamini/one_evaler/one_evaler.py | 70 +++++++++++++++++++++++++ lamini/one_evaler/one_evaler_test.py | 14 +++++ pyproject.toml | 3 +- 13 files changed, 388 insertions(+), 38 deletions(-) create mode 100644 lamini/api/utils/sql_completion.py create mode 100644 lamini/api/utils/sql_token_cache.py create mode 100644 lamini/one_evaler/one_evaler.py create mode 100644 lamini/one_evaler/one_evaler_test.py diff --git a/lamini/__init__.py b/lamini/__init__.py index cff7699..ced463b 100644 --- a/lamini/__init__.py +++ b/lamini/__init__.py @@ -20,7 +20,7 @@ os.environ.get("GATE_PIPELINE_BATCH_COMPLETIONS", False) ) -__version__ = "3.1.3" +__version__ = "3.2.3" # isort: off @@ -30,9 +30,8 @@ from lamini.api.model_downloader import ModelDownloader from lamini.api.model_downloader import ModelType from lamini.api.model_downloader import DownloadedModel -from lamini.classify.lamini_classifier import LaminiClassifier from lamini.generation.generation_node import GenerationNode from lamini.generation.generation_pipeline import GenerationPipeline from lamini.generation.base_prompt_object import PromptObject from lamini.generation.split_response_node import SplitResponseNode -from lamini.api.streaming_completion import StreamingCompletion +from lamini.api.streaming_completion import StreamingCompletion \ No newline at end of file diff --git a/lamini/api/lamini.py b/lamini/api/lamini.py index 82e30e2..976a328 100644 --- a/lamini/api/lamini.py +++ b/lamini/api/lamini.py @@ -1,3 +1,4 @@ +import base64 import enum import json import logging @@ -13,6 +14,8 @@ from lamini.api.rest_requests import get_version, make_web_request from lamini.api.train import Train from lamini.api.utils.completion import Completion +from lamini.api.utils.sql_completion import SQLCompletion +from lamini.api.utils.sql_token_cache import SQLTokenCache from lamini.api.utils.upload_client import upload_to_blob from lamini.error.error import DownloadingModelError @@ -59,6 +62,8 @@ def __init__( self.api_key = api_key self.api_url = api_url self.completion = Completion(api_key, api_url) + self.sql_completion = SQLCompletion(api_key, api_url) + self.sql_token_cache = SQLTokenCache(api_key, api_url) self.trainer = Train(api_key, api_url) self.upload_file_path = None self.upload_base_path = None @@ -77,6 +82,23 @@ def version(self) -> str: """ return get_version(self.api_key, self.api_url, self.config) + def generate_sql( + self, + prompt: Union[str, List[str]], + cache_id: str, + model_name: Optional[str] = None, + max_tokens: Optional[int] = None, + max_new_tokens: Optional[int] = None, + ) -> Union[str, Dict[str, Any]]: + result = self.sql_completion.generate( + prompt=prompt, + cache_id=cache_id, + model_name=model_name or self.model_name, + max_tokens=max_tokens, + max_new_tokens=max_new_tokens, + ) + return result + def generate( self, prompt: Union[str, List[str]], @@ -181,6 +203,7 @@ async def async_generate( req_data = self.completion.make_llm_req_map( prompt=prompt, + cache_id=cache_id, model_name=model_name or self.model_name, output_type=output_type, max_tokens=max_tokens, @@ -424,6 +447,53 @@ def download_model( INTERVAL_SECONDS = 1 time.sleep(INTERVAL_SECONDS) + def add_sql_token_cache( + self, + col_val_file: Optional[str] = None, + wait: bool = False, + wait_time_seconds: int = 600, + ): + col_val_str = None + + if col_val_file: + with open(col_val_file, 'r') as f: + col_vals = json.load(f) + # TODO: in another PR, limit size of col_vals dict + col_val_str = json.dumps(col_vals) + + start_time = time.time() + + while True: + res = self.sql_token_cache.add_token_cache( + base_model_name=self.model_name, + col_vals=col_val_str, + ) + + if not wait: + return res + if res["status"] == "done": + return res + elif res["status"] == "failed": + raise Exception("SQL token cache build failed") + + elapsed_time = time.time() - start_time + if elapsed_time > wait_time_seconds: + return res + INTERVAL_SECONDS = 1 + time.sleep(INTERVAL_SECONDS) + + def delete_sql_token_cache(self, cache_id): + while True: + res = self.sql_token_cache.delete_token_cache(cache_id) + + if res["status"] == "done": + return res + elif res["status"] == "failed": + raise Exception("SQL token cache deletion failed") + + INTERVAL_SECONDS = 1 + time.sleep(INTERVAL_SECONDS) + def list_models(self) -> List[DownloadedModel]: return self.model_downloader.list() diff --git a/lamini/api/model_downloader.py b/lamini/api/model_downloader.py index 31f2a36..18129b0 100644 --- a/lamini/api/model_downloader.py +++ b/lamini/api/model_downloader.py @@ -1,9 +1,10 @@ +""" +A class to handle downloading models from the Lamini Platform. +""" + import enum -from typing import List, Union +from typing import List -import lamini -import numpy as np -from lamini.api.lamini_config import get_config, get_configured_key, get_configured_url from lamini.api.rest_requests import make_web_request @@ -58,7 +59,7 @@ def __init__( api_url: str, ): self.api_key = api_key - self.api_endpoint = api_url + "/v1/downloaded_models/" + self.api_endpoint = api_url + "/v1alpha/downloaded_models/" def download(self, hf_model_name: str, model_type: ModelType) -> DownloadedModel: """Request to Lamini platform for an embedding encoding of the provided diff --git a/lamini/api/rest_requests.py b/lamini/api/rest_requests.py index ad0dd52..2db611a 100644 --- a/lamini/api/rest_requests.py +++ b/lamini/api/rest_requests.py @@ -11,10 +11,12 @@ APIUnprocessableContentError, AuthenticationError, DownloadingModelError, - ModelNotFound, + DuplicateResourceError, + ModelNotFoundError, RateLimitError, RequestTimeoutError, UnavailableResourceError, + ProjectNotFoundError, UserError, ) @@ -174,24 +176,33 @@ async def handle_error(resp: aiohttp.ClientResponse) -> None: Raises ------ - ModelNotFound - Raises from 594 - - RateLimitError - Raises from 429 + UserError + Raises from 400 AuthenticationError Raises from 401 - UserError - Raises from 400 - APIUnprocessableContentError Raises from 422 + RateLimitError + Raises from 429 + + DuplicateResourceError + Raises from 497 + + JobNotFoundError + Raises from 498 + + ProjectNotFoundError + Raises from 499 + UnavailableResourceError Raises from 503 + ModelNotFoundError + Raises from 594 + APIError Raises from 200 @@ -206,7 +217,21 @@ async def handle_error(resp: aiohttp.ClientResponse) -> None: json_response = await resp.json() except Exception: json_response = {} - raise ModelNotFound(json_response.get("detail", "ModelNotFound")) + raise ModelNotFoundError(json_response.get("detail", "ModelNotFound")) + if resp.status == 499: + try: + json_response = await resp.json() + except Exception: + json_response = {} + raise ProjectNotFoundError(json_response.get("detail", "ProjectNotFoundError")) + if resp.status == 497: + try: + json_response = await resp.json() + except Exception: + json_response = {} + raise DuplicateResourceError( + json_response.get("detail", "DuplicateResourceError") + ) if resp.status == 429: try: json_response = await resp.json() @@ -253,7 +278,7 @@ async def handle_error(resp: aiohttp.ClientResponse) -> None: def make_web_request( - key: str, url: str, http_method: str, json: Optional[Dict[str, Any]] = None + key: str, url: str, http_method: str, json: Optional[Dict[str, Any]] = None, stream: bool = False ) -> Dict[str, Any]: """Execute a web request @@ -288,7 +313,7 @@ def make_web_request( HTTPError Raised from many possible reasons: if resp.status_code == 594: - ModelNotFound + ModelNotFoundError if resp.status_code == 429: RateLimitError if resp.status_code == 401: @@ -326,10 +351,14 @@ def make_web_request( pass if http_method == "post": resp = requests.post(url=url, headers=headers, json=json) + elif http_method == "get" and stream: + resp = requests.get(url=url, headers=headers, stream=True) elif http_method == "get": resp = requests.get(url=url, headers=headers) + elif http_method == "delete": + resp = requests.delete(url=url, headers=headers) else: - raise Exception("http_method must be 'post' or 'get'") + raise Exception("http_method must be 'post' or 'get' or 'delete'") try: check_version(resp) resp.raise_for_status() @@ -339,7 +368,23 @@ def make_web_request( json_response = resp.json() except Exception: json_response = {} - raise ModelNotFound(json_response.get("detail", "ModelNameError")) + raise ModelNotFoundError(json_response.get("detail", "ModelNameError")) + if resp.status_code == 499: + try: + json_response = resp.json() + except Exception: + json_response = {} + raise ProjectNotFoundError( + json_response.get("detail", "ProjectNotFoundError") + ) + if resp.status_code == 497: + try: + json_response = resp.json() + except Exception: + json_response = {} + raise DuplicateResourceError( + json_response.get("detail", "DuplicateResourceError") + ) if resp.status_code == 429: try: json_response = resp.json() @@ -401,4 +446,7 @@ def make_web_request( raise APIError("500 Internal Server Error") raise APIError(f"API error {description}") - return resp.json() + if stream: + return resp + else: + return resp.json() diff --git a/lamini/api/utils/completion.py b/lamini/api/utils/completion.py index a90e3cf..23947c4 100644 --- a/lamini/api/utils/completion.py +++ b/lamini/api/utils/completion.py @@ -139,6 +139,7 @@ def make_llm_req_map( self, model_name: str, prompt: Union[str, List[str]], + cache_id: Optional[str] = None, output_type: Optional[dict] = None, max_tokens: Optional[int] = None, max_new_tokens: Optional[int] = None, @@ -186,4 +187,6 @@ def make_llm_req_map( req_data["max_tokens"] = max_tokens if max_new_tokens is not None: req_data["max_new_tokens"] = max_new_tokens + if cache_id is not None: + req_data["cache_id"] = cache_id return req_data diff --git a/lamini/api/utils/sql_completion.py b/lamini/api/utils/sql_completion.py new file mode 100644 index 0000000..31e8c8d --- /dev/null +++ b/lamini/api/utils/sql_completion.py @@ -0,0 +1,40 @@ +from typing import Any, Dict, List, Optional, Union + +import aiohttp +import lamini +from lamini.api.lamini_config import get_config, get_configured_key, get_configured_url +from lamini.api.rest_requests import make_async_web_request, make_web_request +from lamini.api.utils.completion import Completion + +class SQLCompletion(Completion): + def __init__(self, api_key, api_url) -> None: + self.config = get_config() + + self.api_key = api_key or lamini.api_key or get_configured_key(self.config) + self.api_url = api_url or lamini.api_url or get_configured_url(self.config) + self.api_prefix = self.api_url + "/v1alpha/" + + def generate( + self, + prompt: Union[str, List[str]], + cache_id: str, + model_name: str, + max_tokens: Optional[int] = None, + max_new_tokens: Optional[int] = None, + ) -> Dict[str, Any]: + req_data = self.make_llm_req_map( + prompt=prompt, + cache_id=cache_id, + model_name=model_name, + max_tokens=max_tokens, + max_new_tokens=max_new_tokens, + ) + resp = make_web_request( + self.api_key, self.api_prefix + "sql", "post", req_data + ) + return resp + + async def async_generate( + self, params: Dict[str, Any], client: aiohttp.ClientSession = None + ) -> Dict[str, Any]: + raise Exception("SQL streaming not implemented") diff --git a/lamini/api/utils/sql_token_cache.py b/lamini/api/utils/sql_token_cache.py new file mode 100644 index 0000000..1230e19 --- /dev/null +++ b/lamini/api/utils/sql_token_cache.py @@ -0,0 +1,47 @@ +from typing import Any, Dict, List, Optional, Union + +import aiohttp +import lamini +from lamini.api.lamini_config import get_config, get_configured_key, get_configured_url +from lamini.api.rest_requests import make_async_web_request, make_web_request + + +class SQLTokenCache: + def __init__(self, api_key, api_url) -> None: + self.config = get_config() + + self.api_key = api_key or lamini.api_key or get_configured_key(self.config) + self.api_url = api_url or lamini.api_url or get_configured_url(self.config) + self.api_prefix = self.api_url + "/v1alpha/" + + def add_token_cache(self, base_model_name, col_vals=None): + req_data = self.make_req_map( + base_model_name, + col_vals=col_vals, + ) + resp = make_web_request( + self.api_key, self.api_prefix + "add_sql_token_cache", "post", req_data + ) + + return resp + + def delete_token_cache(self, cache_id): + resp = make_web_request( + self.api_key, self.api_prefix + "sql_token_cache/" + cache_id, "delete" + ) + + return resp + + def make_req_map( + self, + base_model_name: str, + col_vals: Optional[dict] = None, + ) -> Dict[str, Any]: + req_data = {} + req_data["base_model_name"] = base_model_name + + if col_vals is not None: + req_data["col_vals"] = col_vals + + return req_data + diff --git a/lamini/api/utils/supported_models.py b/lamini/api/utils/supported_models.py index 88d4cd8..2bd4f61 100644 --- a/lamini/api/utils/supported_models.py +++ b/lamini/api/utils/supported_models.py @@ -28,4 +28,8 @@ TINY_MISTRAL, ] -PROD = [LLAMA_31_8B_INST, LLAMA_32_3B_INST, MISTRAL_7B_INST_V03, TINY_MISTRAL] +PROD = [TINY_MISTRAL, LLAMA_31_8B_INST, LLAMA_32_3B_INST, MISTRAL_7B_INST_V03] + +SENTENCE_TRANSFORMERS = "sentence-transformers/all-MiniLM-L6-v2" + +EMBEDDING_MODELS = [SENTENCE_TRANSFORMERS] diff --git a/lamini/classify/lamini_classifier.py b/lamini/classify/lamini_classifier.py index 98a950b..c18078a 100644 --- a/lamini/classify/lamini_classifier.py +++ b/lamini/classify/lamini_classifier.py @@ -5,7 +5,7 @@ from lamini.api.lamini_config import get_config, get_configured_key, get_configured_url from lamini.api.rest_requests import make_web_request from lamini.api.utils.supported_models import LLAMA_31_8B_INST - +import os logger = logging.getLogger(__name__) @@ -25,29 +25,21 @@ def __init__( self.classifier_name = classifier_name self.model_name = model_name self.classifier_id = None - self.initialize_job_id = None self.train_job_id = None - def initialize(self, classes: dict): + def initialize(self, classes: dict, examples: dict): resp = make_web_request( self.api_key, self.api_prefix + f"/initialize", "post", { "classes": classes, + "examples": examples, "name": self.classifier_name, "model_name": self.model_name, }, ) - self.initialize_job_id = resp["job_id"] - return resp - - def initialize_status(self): - resp = make_web_request( - self.api_key, - self.api_url + f"/v1/data_generation/{self.initialize_job_id}/status", - "get", - ) + self.train_job_id = resp["job_id"] return resp def train(self): @@ -103,3 +95,52 @@ def classify( params, ) return resp + + def download_dataset(self, dataset_name: str, output_dir_path: str = None): + """Download a dataset and save it to disk""" + project_name = self.classifier_name + url = f"{self.api_prefix}/{project_name}/{dataset_name}/download" + filename = f"{dataset_name}.jsonl" + + # Make the request to the API + resp = make_web_request( + self.api_key, + url, + "get", + stream=True # Enable streaming for large files + ) + + # Check if the request was successful + if resp.status_code != 200: + logger.error(f"Failed to download file: {resp.status_code} - {resp.text}") + resp.raise_for_status() # Raise exception for non-200 status codes + + # Set default output path if not provided + if output_dir_path is None: + output_dir_path = "./" + else: + # Ensure directories in output_dir_path exist + os.makedirs(os.path.dirname(output_dir_path), exist_ok=True) + + # Set full path to the output file + output_path = os.path.join(output_dir_path, filename) + logger.debug(f"Downloading to {output_path}") + + # Write the file content to disk + with open(output_path, 'wb') as f: + for chunk in resp.iter_content(chunk_size=8192): + if chunk: # Filter out keep-alive chunks + f.write(chunk) + + logger.debug(f"File successfully downloaded to {output_path}") + return output_path + + def delete_dataset(self, dataset_name: str): + project_name = self.classifier_name + url = f"{self.api_prefix}/{project_name}/{dataset_name}" + resp = make_web_request( + self.api_key, + url, + "delete", + ) + return resp diff --git a/lamini/error/error.py b/lamini/error/error.py index aedcd17..be0f3af 100644 --- a/lamini/error/error.py +++ b/lamini/error/error.py @@ -6,10 +6,22 @@ def __init__( super(LaminiError, self).__init__(message) -class ModelNotFound(LaminiError): +class ProjectNotFoundError(LaminiError): + """The project was not found in the database.""" + + +class DuplicateResourceError(LaminiError): + """The project was not found in the database.""" + + +class ModelNotFoundError(LaminiError): """The model name is invalid. Make sure it's a valid model in Huggingface or a finetuned model""" +class JobNotFoundError(LaminiError): + """No jobs were found which match the specified criteria.""" + + class APIError(LaminiError): """There is an internal error in the Lamini API""" diff --git a/lamini/one_evaler/one_evaler.py b/lamini/one_evaler/one_evaler.py new file mode 100644 index 0000000..0e15dae --- /dev/null +++ b/lamini/one_evaler/one_evaler.py @@ -0,0 +1,70 @@ +import logging +from typing import Dict, List, Optional, Union +import uuid + +import lamini +from lamini.api.lamini_config import get_config, get_configured_key, get_configured_url +from lamini.api.rest_requests import make_web_request +from lamini.api.utils.supported_models import LLAMA_31_8B_INST + +logger = logging.getLogger(__name__) + +class LaminiOneEvaler: + """ + Lamini One Evaler SDK pkg. + """ + def __init__( + self, + test_model_id: str, + eval_data: List[Dict[str, str]], + test_eval_type: str, + eval_data_id: Optional[str] = '', + base_model_id: Optional[str] = '', + base_eval_type: Optional[str] = 'classifier', + sbs: bool= False, + fuzzy: bool = False, + api_key: Optional[str] = None, + api_url: Optional[str] = None, + ): + self.config = get_config() + self.api_key = api_key or lamini.api_key or get_configured_key(self.config) + self.api_url = api_url or lamini.api_url or get_configured_url(self.config) + self.api_prefix = self.api_url + "/v1/" + self.test_model_id = test_model_id + self.eval_data = eval_data + if not eval_data_id: + self.eval_data_id=str(uuid.uuid4()) + else: + self.eval_data_id = eval_data_id + self.test_eval_type = test_eval_type + self.base_model_id=base_model_id + self.base_eval_type=base_eval_type + #TEMP restriction + if self.test_eval_type!='classifier' or self.base_eval_type!='classifier': + raise ValueError("Currently only classifier can use one eval.") + self.sbs=sbs + self.fuzzy=fuzzy + + def run(self): + """ + Run Lamini One Evaler. + Currently only support classifier. + """ + resp = make_web_request( + self.api_key, + self.api_prefix + f"eval/run", + "post", + { + "test_model_id": self.test_model_id, + "test_eval_type": self.test_eval_type, + "eval_data":self.eval_data, + "eval_data_id":self.eval_data_id, + "base_model_id":self.base_model_id, + "base_eval_type": self.base_eval_type, + "sbs": self.sbs, + "fuzzy_comparison":self.fuzzy + + }, + ) + self.eval_job_id = resp["eval_job_id"] + return resp diff --git a/lamini/one_evaler/one_evaler_test.py b/lamini/one_evaler/one_evaler_test.py new file mode 100644 index 0000000..803aaed --- /dev/null +++ b/lamini/one_evaler/one_evaler_test.py @@ -0,0 +1,14 @@ +import one_evaler +import pytest + +@pytest.skip("real run, temp. ban") +def test_one_evaler(): + eval=one_evaler.LaminiOneEvaler(api_key='', + test_model_id="8ee3050e-486b-4ac4-9588-7e0b8cad3499", + eval_data=[{'input':'dog', 'target':'cat'}], + test_eval_type='classifier' + ) + result=eval.run() + + assert result['status']=='COMPLETED' + assert result['predictions']==[{'base_output': None, 'input': 'dog', 'target': 'cat', 'test_output': 'out_of_scope'}] diff --git a/pyproject.toml b/pyproject.toml index b8b2d7b..fab8d55 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "lamini" -version = "3.1.3" +version = "3.2.3" authors = [ { name="Lamini", email="info@lamini.ai" }, ] @@ -46,6 +46,7 @@ packages = [ "lamini.evaluators.helm", "lamini.evaluators.utils", "lamini.index", + "lamini.one_evaler", ] [tool.setuptools.package-data]